diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java index 3840774313e..ad6e3d24445 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java @@ -7,8 +7,6 @@ import java.util.List; import java.util.Locale; -import net.sourceforge.pmd.annotation.DeprecatedUntil700; - /** * * @author Zev Blut zb@ubit.com @@ -17,7 +15,6 @@ * @deprecated Use an {@link AnyTokenizer} instead, it's basically as powerful. */ @Deprecated -@DeprecatedUntil700 public abstract class AbstractTokenizer implements Tokenizer { // FIXME depending on subclasses to assign local vars is rather fragile - diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java index 76e0e4ce5ed..d476af6e006 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java @@ -18,7 +18,7 @@ * Higher-quality lexers should be implemented with a lexer generator. * *
In PMD 7, this replaces AbstractTokenizer, which provided nearly
- * no more functionality and whose API was hard to update.
+ * no more functionality.
*/
public class AnyTokenizer implements Tokenizer {
@@ -61,7 +61,7 @@ private static String eolCommentFragment(String start) {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
- StringBuilder text = sourceCode.getCodeBuffer();
+ CharSequence text = sourceCode.getCodeBuffer();
Matcher matcher = pattern.matcher(text);
int lineNo = 1;
int lastLineStart = 0;
diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java
index 4e6ee6e9a77..330f6da5c29 100644
--- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java
+++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java
@@ -7,14 +7,11 @@
import static net.sourceforge.pmd.util.CollectionUtil.listOf;
import static org.junit.Assert.assertEquals;
+import java.util.ArrayList;
import java.util.List;
-import java.util.stream.Collectors;
-import org.checkerframework.checker.nullness.qual.NonNull;
import org.junit.Test;
-import net.sourceforge.pmd.internal.util.IteratorUtil;
-
public class AnyTokenizerTest {
@Test
@@ -33,13 +30,13 @@ public void testStringEscape() {
public void testMultilineString() {
AnyTokenizer tokenizer = new AnyTokenizer("//");
Tokens tokens = compareResult(tokenizer, "a = \"oo\n\";", listOf("a", "=", "\"oo\n\"", ";", "EOF"));
- TokenEntry string = IteratorUtil.getNth(tokens.iterator(), 2);
+ TokenEntry string = tokens.getTokens().get(2);
assertEquals("\"oo\n\"", getTokenImage(string));
assertEquals(1, string.getBeginLine());
assertEquals(5, string.getBeginColumn());
assertEquals(2, string.getEndColumn()); // ends on line 2
- TokenEntry semi = IteratorUtil.getNth(tokens.iterator(), 3);
+ TokenEntry semi = tokens.getTokens().get(3);
assertEquals(";", getTokenImage(semi));
assertEquals(2, semi.getBeginLine());
assertEquals(2, semi.getBeginColumn());
@@ -50,15 +47,17 @@ private Tokens compareResult(AnyTokenizer tokenizer, String source, List