Skip to content

Commit

Permalink
fix parsing characters
Browse files Browse the repository at this point in the history 
Fix literal space #Add missing character names like backspace or alert
  • Loading branch information
@jcubic
jcubic committed Apr 18, 2020
1 parent 3ce935b commit 8028211
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 13 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# [LIPS is Pretty Simple](https://jcubic.github.io/lips/) - Scheme based Powerful LISP

[![npm](https://img.shields.io/badge/npm-DEV-blue.svg)](https://www.npmjs.com/package/@jcubic/lips)
[![travis](https://travis-ci.org/jcubic/lips.svg?branch=devel&f7e0e6d64d7500f633db65bde8bef1a999ba46e9)](https://travis-ci.org/jcubic/lips)
[![travis](https://travis-ci.org/jcubic/lips.svg?branch=devel&3ce935ba06e3818e9fd8ba76cfa0b9ffbbe385f4)](https://travis-ci.org/jcubic/lips)
[![Coverage Status](https://coveralls.io/repos/github/jcubic/lips/badge.svg?branch=devel&e16c39d35b2647b16bfa848e48a6c3bb)](https://coveralls.io/github/jcubic/lips?branch=devel)


Expand Down
24 changes: 18 additions & 6 deletions dist/lips.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
* Copyright (c) 2014-present, Facebook, Inc.
* released under MIT license
*
* build: Sat, 18 Apr 2020 08:08:11 +0000
* build: Sat, 18 Apr 2020 08:38:56 +0000
*/
(function () {
'use strict';
Expand Down Expand Up @@ -1158,7 +1158,8 @@
var re_re = /^\/((?:\\\/|[^/]|\[[^\]]*\/[^\]]*\])+)\/([gimy]*)$/;
var int_re = /^(?:#x[-+]?[0-9a-f]+|#o[-+]?[0-7]+|#b[-+]?[01]+|[-+]?[0-9]+)$/i;
var float_re = /^([-+]?([0-9]+([eE][-+]?[0-9]+)|(\.[0-9]+|[0-9]+\.[0-9]+)([eE][-+]?[0-9]+)?)|[0-9]+\.)$/;
var char_re = /^#\\(?:newline|space|.)$/i; // (int | flat) ? ([-+]
var character_symbols = ['alarm', 'backspace', 'delete', 'escape', 'newline', 'null', 'return', 'space', 'tab'].join('|');
var char_re = new RegExp("^#\\\\(?:".concat(character_symbols, "|[\\s\\S])$"), 'i'); // (int | flat) ? ([-+]

var complex_re = /^((?:(?:[-+]?[0-9]+(?:[eE][-+]?[0-9]+)?)|(?:[-+]?(?:(?:\.[0-9]+|[0-9]+\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\.))(?=[+-]|i))?((?:[-+]?[0-9]+(?:[eE][-+]?[0-9]+)?)|(?:[-+]?(?:(?:\.[0-9]+|[0-9]+\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\.))i|-i$/;
var rational_re = /^[-+]?[0-9]+\/[0-9]+$/;
Expand Down Expand Up @@ -1298,7 +1299,7 @@
var tokens = specials.names().sort(function (a, b) {
return b.length - a.length || a.localeCompare(b);
}).map(escape_regex).join('|');
return new RegExp("(#\\\\(?:newline|space|\\S)|#f|#t|#[xbo][0-9a-f]+(?=[\\s()]|$)|[0-9]+/[0-9]+|\\[|\\]|\\(|\\)|;.*|(?:(?:[-+]?(?:(?:\\.[0-9]+|[0-9]+\\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\\.)[0-9]i)|\\n|\\.{2,}|(?!#:)(?:".concat(tokens, ")|[^(\\s)[\\]]+)"), 'gim');
return new RegExp("(#\\\\(?:".concat(character_symbols, "|[\\s\\S])|#f|#t|#[xbo][0-9a-f]+(?=[\\s()]|$)|[0-9]+/[0-9]+|\\[|\\]|\\(|\\)|;.*|(?:(?:[-+]?(?:(?:\\.[0-9]+|[0-9]+\\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\\.)[0-9]i)|\\n|\\.{2,}|(?!#:)(?:").concat(tokens, ")|[^(\\s)[\\]]+)"), 'gim');
/*
return new RegExp(`("(?:\\\\[\\S\\s]|[^"])*"|#\\\\(?:newline|space|\\S)|#f|#t|#[xbo][0-9a-f]+(?=[\\s()]|$)|[0-9]+/[0-9]+|\\/(?! )[^\\n\\/\\\\]*(?:\\\\[\\S\\s][^\\n\\/\\\\]*)*\\/[gimy]*(?=\\s|\\(|\\)|\\]|\\[|$)|\\[|\\]|\\(|\\)|"(?:\\\\[\\S\\s]|[^"])+|\\n|(?:\\\\[\\S\\s]|[^"])*"|;.*|(?:(?:[-+]?(?:(?:\\.[0-9]+|[0-9]+\\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\\.)[0-9]i)|\\.{2,}|(?!#:)(?:${tokens})|[^(\\s)[\\]]+)`, 'gim');
*/
Expand Down Expand Up @@ -1360,7 +1361,8 @@
return;
}

string.split(tokens_re).filter(Boolean).forEach(function (string) {
var parts = string.split(tokens_re).filter(Boolean);
parts.forEach(function (string) {
var token = {
col: col,
line: line,
Expand Down Expand Up @@ -1409,6 +1411,11 @@

if (!ret || typeof ret.token !== 'string') {
throw new Error('[tokenize] Invalid formatter wrong return object');
} // we don't want literal space character to be trimmed


if (ret.token === '#\\ ') {
return ret.token;
}

return ret.token.trim();
Expand Down Expand Up @@ -3914,9 +3921,14 @@
}

LCharacter.names = {
'space': ' ',
'alarm': '\x07',
'backspace': '\x08',
'delete': '\x7F',
'escape': '\x1B',
'newline': '\n',
'null': '\x00',
'return': '\r',
'space': ' ',
'tab': '\t'
};
LCharacter.rev_names = {};
Expand Down Expand Up @@ -8265,7 +8277,7 @@
var lips = {
version: 'DEV',
banner: banner,
date: 'Sat, 18 Apr 2020 08:08:11 +0000',
date: 'Sat, 18 Apr 2020 08:38:56 +0000',
exec: exec,
parse: parse,
tokenize: tokenize,
Expand Down
4 changes: 2 additions & 2 deletions dist/lips.min.js
View file

Large diffs are not rendered by default.

21 changes: 17 additions & 4 deletions src/lips.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,10 @@ You can also use (help name) to display help for specic function or macro.
var re_re = /^\/((?:\\\/|[^/]|\[[^\]]*\/[^\]]*\])+)\/([gimy]*)$/;
var int_re = /^(?:#x[-+]?[0-9a-f]+|#o[-+]?[0-7]+|#b[-+]?[01]+|[-+]?[0-9]+)$/i;
var float_re = /^([-+]?([0-9]+([eE][-+]?[0-9]+)|(\.[0-9]+|[0-9]+\.[0-9]+)([eE][-+]?[0-9]+)?)|[0-9]+\.)$/;
var char_re = /^#\\(?:newline|space|.)$/i;
var character_symbols = [
'alarm', 'backspace', 'delete', 'escape', 'newline', 'null', 'return', 'space', 'tab'
].join('|');
var char_re = new RegExp(`^#\\\\(?:${character_symbols}|[\\s\\S])$`, 'i');
// (int | flat) ? ([-+]
var complex_re = /^((?:(?:[-+]?[0-9]+(?:[eE][-+]?[0-9]+)?)|(?:[-+]?(?:(?:\.[0-9]+|[0-9]+\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\.))(?=[+-]|i))?((?:[-+]?[0-9]+(?:[eE][-+]?[0-9]+)?)|(?:[-+]?(?:(?:\.[0-9]+|[0-9]+\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\.))i|-i$/;
var rational_re = /^[-+]?[0-9]+\/[0-9]+$/;
Expand Down Expand Up @@ -292,7 +295,7 @@ You can also use (help name) to display help for specic function or macro.
var tokens = specials.names()
.sort((a, b) => b.length - a.length || a.localeCompare(b))
.map(escape_regex).join('|');
return new RegExp(`(#\\\\(?:newline|space|\\S)|#f|#t|#[xbo][0-9a-f]+(?=[\\s()]|$)|[0-9]+/[0-9]+|\\[|\\]|\\(|\\)|;.*|(?:(?:[-+]?(?:(?:\\.[0-9]+|[0-9]+\\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\\.)[0-9]i)|\\n|\\.{2,}|(?!#:)(?:${tokens})|[^(\\s)[\\]]+)`, 'gim');
return new RegExp(`(#\\\\(?:${character_symbols}|[\\s\\S])|#f|#t|#[xbo][0-9a-f]+(?=[\\s()]|$)|[0-9]+/[0-9]+|\\[|\\]|\\(|\\)|;.*|(?:(?:[-+]?(?:(?:\\.[0-9]+|[0-9]+\\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\\.)[0-9]i)|\\n|\\.{2,}|(?!#:)(?:${tokens})|[^(\\s)[\\]]+)`, 'gim');
/*
return new RegExp(`("(?:\\\\[\\S\\s]|[^"])*"|#\\\\(?:newline|space|\\S)|#f|#t|#[xbo][0-9a-f]+(?=[\\s()]|$)|[0-9]+/[0-9]+|\\/(?! )[^\\n\\/\\\\]*(?:\\\\[\\S\\s][^\\n\\/\\\\]*)*\\/[gimy]*(?=\\s|\\(|\\)|\\]|\\[|$)|\\[|\\]|\\(|\\)|"(?:\\\\[\\S\\s]|[^"])+|\\n|(?:\\\\[\\S\\s]|[^"])*"|;.*|(?:(?:[-+]?(?:(?:\\.[0-9]+|[0-9]+\\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|[0-9]+\\.)[0-9]i)|\\.{2,}|(?!#:)(?:${tokens})|[^(\\s)[\\]]+)`, 'gim');
*/
Expand Down Expand Up @@ -344,7 +347,8 @@ You can also use (help name) to display help for specic function or macro.
line += (string.match("\n") || []).length;
return;
}
string.split(tokens_re).filter(Boolean).forEach(function(string) {
var parts = string.split(tokens_re).filter(Boolean);
parts.forEach(function(string) {
var token = {
col,
line,
Expand Down Expand Up @@ -386,6 +390,10 @@ You can also use (help name) to display help for specic function or macro.
if (!ret || typeof ret.token !== 'string') {
throw new Error('[tokenize] Invalid formatter wrong return object');
}
// we don't want literal space character to be trimmed
if (ret.token === '#\\ ') {
return ret.token;
}
return ret.token.trim();
}).filter(function(token) {
return token && !token.match(/^;/);
Expand Down Expand Up @@ -2353,9 +2361,14 @@ You can also use (help name) to display help for specic function or macro.
}
}
LCharacter.names = {
'space': ' ',
'alarm': '\x07',
'backspace': '\x08',
'delete': '\x7F',
'escape': '\x1B',
'newline': '\n',
'null': '\x00',
'return': '\r',
'space': ' ',
'tab': '\t'
};
LCharacter.rev_names = {};
Expand Down

0 comments on commit 8028211

Please sign in to comment.