LANG: ICharSource changes. +LangPartitionScannerTest.

plugin_ide.core.tests/src-lang/melnorme/lang/ide/core/text/CharacterScanner_ReaderHelper_Test.java

@@ -16,13 +16,14 @@
 import org.junit.Test;
 
 import melnorme.lang.utils.parse.ICharSource;
+import melnorme.lang.utils.parse.ICharacterReader;
 import melnorme.lang.utils.parse.ParseSource_Test;
 
 public class CharacterScanner_ReaderHelper_Test {
 
 	public static class ReaderHelper_ParseSource_Test extends ParseSource_Test {
 		@Override
-		protected ICharSource<?> createParseSource(String source) {
+		protected ICharacterReader createParseSource(String source) {
 			BufferedRuleBasedScannerExt charScanner = new BufferedRuleBasedScannerExt();
 			charScanner.setRange(new Document(source), 0, source.length());
 			return new CharacterScanner_ReaderHelper(charScanner); 
@@ -51,16 +52,16 @@ protected void checkBufferedCount(ICharSource<?> parseSource, int expected) {
 		assertTrue(charScanner.lookaheadString(4, 0).equals(""));
 		assertTrue(reader.lookahead() == 'a');
 
-		reader.consume();
+		reader.consume2();
 		assertTrue(charScanner.getOffset() == 1);
 		assertTrue(charScanner.lookahead(0) == 'b');
 		assertTrue(charScanner.lookaheadString(0, 0).equals(""));
 		assertTrue(charScanner.lookaheadString(0, 2).equals("bc"));
 		assertTrue(charScanner.lookaheadString(1, 2).equals("cd"));
 		assertTrue(charScanner.lookaheadString(3, 0).equals(""));
 
-		reader.consume();
-		reader.consume();
+		reader.consume2();
+		reader.consume2();
 		assertTrue(charScanner.getOffset() == 3);
 		assertTrue(charScanner.lookahead(0) == 'd');
 		assertTrue(reader.lookahead() == 'd');
@@ -71,7 +72,7 @@ protected void checkBufferedCount(ICharSource<?> parseSource, int expected) {
 
 		reader = new CharacterScanner_ReaderHelper(charScanner);
 		assertTrue(charScanner.lookahead(0) == 'a');
-		assertTrue(reader.consume() == 'a');
+		assertTrue(reader.consume2() == 'a');
 		assertTrue(charScanner.lookahead(0) == 'b');
 		assertTrue(charScanner.getOffset() == 1);
 		assertTrue(reader.lookahead(1) == 'c');

plugin_ide.core.tests/src-lang/melnorme/lang/ide/core/text/LangPartitionScannerTest.java

@@ -0,0 +1,76 @@
+package melnorme.lang.ide.core.text;
+/*******************************************************************************
+ * Copyright (c) 2015 Bruno Medeiros and other Contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Bruno Medeiros - initial API and implementation
+ *******************************************************************************/
+
+
+
+import static melnorme.utilbox.core.Assert.AssertNamespace.assertNotNull;
+import static melnorme.utilbox.core.Assert.AssertNamespace.assertTrue;
+import static melnorme.utilbox.core.CoreUtil.downCast;
+
+import org.eclipse.jface.text.BadPositionCategoryException;
+import org.eclipse.jface.text.Document;
+import org.eclipse.jface.text.Position;
+import org.eclipse.jface.text.TypedPosition;
+import org.eclipse.jface.text.rules.FastPartitioner;
+
+import melnorme.utilbox.misc.ArrayUtil;
+import melnorme.utilbox.tests.CommonTest;
+
+public class LangPartitionScannerTest extends CommonTest {
+
+	protected static final String NL = "\n";
+	protected Document document;
+	protected FastPartitioner fp;
+	protected boolean recreateDocSetup = true;
+
+	public LangPartitionScannerTest() {
+		super();
+	}
+
+
+	protected void testPartitions(String source, Enum<?>... expectedPositions) throws BadPositionCategoryException {
+		testPartitions(source, ArrayUtil.map(expectedPositions, 
+			(enm) -> enm.toString(), String.class));
+	}
+
+	protected void testPartitions(String source, String... expectedPositions) throws BadPositionCategoryException {
+		Position[] positions = calculatePartitions(source);
+		checkPositions(positions, expectedPositions);
+	}
+
+	protected Position[] calculatePartitions(String docContents) throws BadPositionCategoryException {
+		setupDocument(docContents);
+		Position[] positions = getPartitionPositions();
+		return positions;
+	}
+	protected void setupDocument(String docContents) {
+		if(recreateDocSetup){ 
+			document = new Document(docContents);
+			fp = LangDocumentPartitionerSetup.getInstance().setupDocument(document);
+		} else {
+			document.set(docContents);
+			assertNotNull(fp);
+		}
+	}
+	protected Position[] getPartitionPositions() throws BadPositionCategoryException {
+		return document.getPositions(fp.getManagingPositionCategories()[0]);
+	}
+
+	protected void checkPositions(Position[] positions, String[] expectedPositions) {
+		assertTrue(positions.length == expectedPositions.length);
+		for (int i = 0; i < positions.length; i++) {
+			TypedPosition position = downCast(positions[i], TypedPosition.class);
+			assertTrue(position.getType() == expectedPositions[i]);
+		}
+	}
+
+}

plugin_ide.core.tests/src-lang/melnorme/lang/ide/core/text/format/LangAutoEditStrategyTest.java

@@ -77,7 +77,7 @@ public boolean parenthesesAsBlocks() {
 	public static final String NEUTRAL_SRCX; 
 
 	static {
-		NEUTRAL_SRCX = MiscUtil.getClassResourceAsString(LangAutoEditStrategyTest.class, "sample_block_code");
+		NEUTRAL_SRCX = MiscUtil.getClassResource(LangAutoEditStrategyTest.class, "sample_block_code");
 	}
 
 	public static final String PENDING_WS1 = "  "; 

plugin_tooling/src-lang/melnorme/lang/tests/CommonLexerRuleTest.java

@@ -0,0 +1,55 @@
+/*******************************************************************************
+ * Copyright (c) 2015 Bruno Medeiros and other Contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Bruno Medeiros - initial API and implementation
+ *******************************************************************************/
+package melnorme.lang.tests;
+
+
+import melnorme.lang.tooling.parser.lexer.ILexingRule;
+import melnorme.lang.utils.parse.StringParseSource;
+
+public abstract class CommonLexerRuleTest extends CommonToolingTest {
+
+	public CommonLexerRuleTest() {
+		super();
+	}
+
+	public void testRule(String source) {
+		testRule(source, true);
+	}
+
+	public void testRule(String source, boolean terminatedSuccessfuly) {
+		testRule(source, source.length());
+		if(terminatedSuccessfuly) {
+			// if rule terminated successfully, adding a suffix will make no diference to token size
+			testRule(source + getRuleNeutralSuffix(), source.length());
+		}
+	}
+
+	protected String getRuleNeutralSuffix() {
+		return "xxxx";
+	}
+
+	public void testRule(String source, int expectedTokenLength) {
+		testRule(createLexingRule(), source, expectedTokenLength);
+	}
+
+	protected abstract ILexingRule createLexingRule();
+
+	public static void testRule(ILexingRule lexRule, String source, int expectedTokenLength) {
+		StringParseSource reader = new StringParseSource(source);
+		boolean isMatch = lexRule.tryMatch(reader);
+
+		assertEquals(isMatch, expectedTokenLength > 0);
+		if(isMatch) {
+			assertEquals(reader.getReadOffset(), expectedTokenLength);
+		}
+	}
+
+}

plugin_tooling/src-lang/melnorme/lang/tooling/ops/util/SourceLinesInfo.java

@@ -51,7 +51,7 @@ protected void consumeNewLine(StringParseSource parser) {
 				return;
 			}
 
-			parser.consume();
+			parser.consume2();
 		}
 
 	}

plugin_tooling/src-lang/melnorme/lang/tooling/parser/lexer/BasicCharSource.java

@@ -23,11 +23,12 @@ public BasicCharSource() {
 	}
 
 	@Override
-	public final int consume() throws EXC {
+	public final char consume2() throws EXC {
 		checkedHasNext = false;
 		int next = lookahead();
+		assertTrue(next >= 0);
 		doConsume();
-		return next;
+		return (char) next;
 	}
 
 	protected abstract void doConsume() throws EXC;

plugin_tooling/src-lang/melnorme/lang/tooling/parser/lexer/CharacterLexingRule.java

@@ -37,7 +37,13 @@ protected boolean consumeBody(ICharacterReader reader) {
 			return reader.tryConsume('\'') || consumeCommonEscape(reader) || consumeUnicodeEscapeSequence(reader);
 		};
 
-		return consumeAnyExceptNullOr(reader, '\'');
+		if(reader.lookaheadIsEOF() || reader.lookahead() == '\'') {
+			return false;
+		}
+		if(reader.lookahead() == 0) {
+			return false; // Should the null character really not be allowed?
+		}
+		return reader.consumeAny();
 	}
 
 	protected boolean consumeEnd(ICharacterReader reader) {
@@ -75,10 +81,11 @@ protected boolean consumeUnicodeEscapeSequence(ICharacterReader reader) {
 				int la = reader.lookahead();
 
 				// This is not accurate to any spec, but is good enough.
-				if(!(la == '{' || la == '}' || isHexDigit(la))) {
+				if(la == '{' || la == '}' || isHexDigit(la)) {
+					reader.consume2();
+				} else {
 					break;
 				}
-				reader.consume();
 			}
 
 			return true;

plugin_tooling/src-lang/melnorme/lang/tooling/parser/lexer/LexingUtils.java

@@ -15,23 +15,20 @@
 
 public class LexingUtils {
 
-	public static <E extends Exception> boolean consumeAnyExceptNullOr(IBasicCharSource<E> reader, int character) 
-			throws E {
-		int ch = reader.lookahead();
-		if(ch != character && ch != 0) {
-			reader.consume();
-			return true;
-		}
-		return false;
+	/** 
+	 * Consume until given delimiter char is found (also included).	 
+	 */
+	public static <E extends Exception> String consumeUntilDelimiter(ICharSource<E> reader, 
+			char delimiter) throws E {
+		return consumeUntilDelimiter(reader, delimiter, delimiter);
 	}
 
-
-	/**
-	 * Consume a string delimited by give delimiter char, 
-	 * with given escapeChar acting a possible escape (use -1 for no escapeChar)
+	/** 
+	 * Consume until given delimiter char is found (also included).
+	 * Given escapeChar can escape itself, and delimiter 	 
 	 */
-	public static <E extends Exception> String consumeDelimitedString(IBasicCharSource<E> reader, 
-			int delimiter, int escapeChar) throws E {
+	public static <E extends Exception> String consumeUntilDelimiter(ICharSource<E> reader, 
+			char delimiter, char escapeChar) throws E {
 		StringBuilder sb = new StringBuilder();
 
 		while(reader.hasCharAhead()) {
@@ -42,11 +39,11 @@ public static <E extends Exception> String consumeDelimitedString(IBasicCharSour
 				break;
 			}
 			else if(consumedChar == escapeChar && reader.hasCharAhead()) {
-				int lookahead = reader.lookahead();
-				if(lookahead == delimiter || lookahead == escapeChar) {
-					
-					char escapedChar = reader.nextChar();
-					sb.append(escapedChar);
+
+				char secondChar = reader.lookaheadChar();
+				if(secondChar == delimiter || secondChar == escapeChar) {
+					reader.nextChar();
+					sb.append(secondChar);
 					continue;
 				}
 			}
@@ -57,12 +54,30 @@ else if(consumedChar == escapeChar && reader.hasCharAhead()) {
 		return sb.toString();
 	}
 
+
+	public static <E extends Exception> void advanceDelimitedString(ICharSource<E> reader, 
+			char delimiter, char escapeChar) throws E {
+
+		while(reader.hasCharAhead()) {
+
+			char consumedChar = reader.nextChar();
+
+			if(consumedChar == delimiter) {
+				break;
+			}
+			else if(consumedChar == escapeChar) {
+				reader.consumeAny(); // consumed escaped char
+			}
+
+		}
+	}
+
 	/* ----------------- whitespace ----------------- */
 
 	public static <E extends Exception> int skipWhitespace(IBasicCharSource<E> reader) throws E {
 		int count = 0;
 		while(Character.isWhitespace(reader.lookahead())) {
-			reader.consume();
+			reader.consume2();
 			count++;
 		}
 		return count;

plugin_tooling/src-lang/melnorme/lang/utils/parse/IBasicCharSource.java

@@ -18,7 +18,15 @@ public interface IBasicCharSource<EXC extends Exception> {
 	int lookahead() throws EXC;
 
 	/** @return the next character in the stream, advancing the stream. */
-	int consume() throws EXC;
+	char consume2() throws EXC;
+
+	default char nextChar() throws EXC {
+		return consume2();
+	}
+
+	default boolean lookaheadIsEOF() throws EXC {
+		return !hasCharAhead();
+	}
 
 	default boolean hasCharAhead() throws EXC {
 		return lookahead() != -1;
@@ -30,18 +38,20 @@ default char lookaheadChar() throws EXC {
 		return (char) la;
 	}
 
-	default char nextChar() throws EXC {
-		int ch = consume();
-		assertTrue(ch != -1);
-		return (char) ch;
-	}
-
 	/* -----------------  ----------------- */
 
 	/** Read next character if it is equal to given character. @return true if a character was read. */
 	default boolean tryConsume(char character) throws EXC {
 		if(lookahead() == character) {
-			consume();
+			consume2();
+			return true;
+		}
+		return false;
+	}
+
+	default boolean consumeAny() throws EXC {
+		if(hasCharAhead()) {
+			consume2();
 			return true;
 		}
 		return false;

plugin_tooling/src-lang/melnorme/lang/utils/parse/ICharSource.java

@@ -36,13 +36,9 @@ default int lookahead() throws EXC {
 
 	/* -----------------  Some common helpers  ----------------- */
 
-	default boolean lookaheadIsEOF() throws EXC {
-		return !hasCharAhead();
-	}
-
 	default void consume(int amount) throws EXC {
 		while(amount-- > 0) {
-			consume();
+			consume2();
 		}
 	}