Merge remote-tracking branch 'LangEclipseIDE/master'

plugin_ide.core/src-lang/melnorme/lang/ide/core/text/StrictDamagerRepairer.java

@@ -0,0 +1,42 @@
+/*******************************************************************************
+ * Copyright (c) 2015 Bruno Medeiros and other Contributors.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Bruno Medeiros - initial API and implementation
+ *******************************************************************************/
+package melnorme.lang.ide.core.text;
+
+import org.eclipse.jface.text.DocumentEvent;
+import org.eclipse.jface.text.IRegion;
+import org.eclipse.jface.text.ITypedRegion;
+import org.eclipse.jface.text.Region;
+import org.eclipse.jface.text.rules.DefaultDamagerRepairer;
+import org.eclipse.jface.text.rules.ITokenScanner;
+
+/**
+ * A strict damager-repairer ensures that a damage region for a document event always starts 
+ * at partition start (or before that)
+ */
+public class StrictDamagerRepairer extends DefaultDamagerRepairer {
+
+	public StrictDamagerRepairer(ITokenScanner scanner) {
+		super(scanner);
+	}
+
+	@Override
+	public IRegion getDamageRegion(ITypedRegion partition, DocumentEvent e, 
+			boolean documentPartitioningChanged) {
+		IRegion region = super.getDamageRegion(partition, e, documentPartitioningChanged);
+
+		int start = region.getOffset();
+		int end = region.getOffset() + region.getLength();
+		start = Math.min(start, partition.getOffset());
+
+		return new Region(start, end - start);
+	}
+
+}

plugin_ide.ui/src-lang/melnorme/lang/ide/ui/text/AbstractSimpleLangSourceViewerConfiguration.java

@@ -130,13 +130,18 @@ protected void handleTokenModified(Token token) {
 			AbstractLangScanner scanner = createScannerFor(Display.getCurrent(), partitionType, tokenRegistry);
 			scanners.add(scanner);
 
-			DefaultDamagerRepairer dr = new DefaultDamagerRepairer(scanner);
+			DefaultDamagerRepairer dr = getDamagerRepairer(scanner, contentType);
 			reconciler.setDamager(dr, contentType);
 			reconciler.setRepairer(dr, contentType);
 		}
 
 	}
 
+	@SuppressWarnings("unused")
+	protected DefaultDamagerRepairer getDamagerRepairer(AbstractLangScanner scanner, String contentType) {
+		return new DefaultDamagerRepairer(scanner);
+	}
+
 	protected void addConfigurationScopedOwned(ISourceViewer sourceViewer, IDisposable tokenStore) {
 		if(sourceViewer instanceof ProjectionViewerExt) {
 			ProjectionViewerExt viewerExt = (ProjectionViewerExt) sourceViewer;

plugin_tooling/src-lang/melnorme/lang/tooling/parser/lexer/LexingUtils.java

@@ -83,6 +83,20 @@ public static <E extends Exception> int skipWhitespace(IBasicCharSource<E> reade
 		return count;
 	}
 
+	public static <E extends Exception> int skipWhitespaceExceptNL(IBasicCharSource<E> reader) throws E {
+		int count = 0;
+		while(Character.isWhitespace(reader.lookahead())) {
+
+			if(reader.lookahead() == '\n' || reader.lookahead() == '\r') {
+				break;
+			}
+
+			reader.consume2();
+			count++;
+		}
+		return count;
+	}
+
 	/* ----------------- Identifier helpers ----------------- */
 
 	public static <E extends Exception> String tryConsumeJavaIdentifier(ICharSource<E> reader) throws E {

plugin_tooling/src-lang/melnorme/lang/utils/parse/ParseSource_Test.java

@@ -154,15 +154,17 @@ public static int testLookAhead(ICharSource<?> parseSource, int expected) throws
 
 	protected void testConsumeDelimitedString(ICharacterReader parseSource, char delimiter, char escapeChar, 
 			String expected) throws Exception {
+
 		CharacterReader_SubReader subReader = new CharacterReader_SubReader(parseSource);
-		LexingUtils.consumeUntilDelimiter(subReader, delimiter, escapeChar);
+		CharacterReader_SubReader subReaderAlt2 = new CharacterReader_SubReader(parseSource);
 
-		OffsetBasedCharacterReader<?> parseSource_ = (OffsetBasedCharacterReader<?>) parseSource;
-		int originalReadOffset = parseSource_.readOffset;
-
-		assertEquals(LexingUtils.consumeUntilDelimiter(parseSource, delimiter, escapeChar), expected);
+		assertEquals(LexingUtils.consumeUntilDelimiter(subReader, delimiter, escapeChar), expected);
 
-		assertTrue(parseSource_.readOffset - originalReadOffset == subReader.readOffset);
+		// ensure advanceDelimitedString reads same number of strings;
+		LexingUtils.advanceDelimitedString(subReaderAlt2, delimiter, escapeChar);
+		assertTrue(subReader.readOffset == subReaderAlt2.readOffset);
+
+		assertEquals(LexingUtils.consumeUntilDelimiter(parseSource, delimiter, escapeChar), expected);
 	}
 
 	@Test