diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java index 3840774313e..ad6e3d24445 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java @@ -7,8 +7,6 @@ import java.util.List; import java.util.Locale; -import net.sourceforge.pmd.annotation.DeprecatedUntil700; - /** * * @author Zev Blut zb@ubit.com @@ -17,7 +15,6 @@ * @deprecated Use an {@link AnyTokenizer} instead, it's basically as powerful. */ @Deprecated -@DeprecatedUntil700 public abstract class AbstractTokenizer implements Tokenizer { // FIXME depending on subclasses to assign local vars is rather fragile - diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java index 76e0e4ce5ed..d476af6e006 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java @@ -18,7 +18,7 @@ * Higher-quality lexers should be implemented with a lexer generator. * *

In PMD 7, this replaces AbstractTokenizer, which provided nearly - * no more functionality and whose API was hard to update. + * no more functionality. */ public class AnyTokenizer implements Tokenizer { @@ -61,7 +61,7 @@ private static String eolCommentFragment(String start) { @Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { - StringBuilder text = sourceCode.getCodeBuffer(); + CharSequence text = sourceCode.getCodeBuffer(); Matcher matcher = pattern.matcher(text); int lineNo = 1; int lastLineStart = 0; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java index 4e6ee6e9a77..330f6da5c29 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/AnyTokenizerTest.java @@ -7,14 +7,11 @@ import static net.sourceforge.pmd.util.CollectionUtil.listOf; import static org.junit.Assert.assertEquals; +import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; -import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; -import net.sourceforge.pmd.internal.util.IteratorUtil; - public class AnyTokenizerTest { @Test @@ -33,13 +30,13 @@ public void testStringEscape() { public void testMultilineString() { AnyTokenizer tokenizer = new AnyTokenizer("//"); Tokens tokens = compareResult(tokenizer, "a = \"oo\n\";", listOf("a", "=", "\"oo\n\"", ";", "EOF")); - TokenEntry string = IteratorUtil.getNth(tokens.iterator(), 2); + TokenEntry string = tokens.getTokens().get(2); assertEquals("\"oo\n\"", getTokenImage(string)); assertEquals(1, string.getBeginLine()); assertEquals(5, string.getBeginColumn()); assertEquals(2, string.getEndColumn()); // ends on line 2 - TokenEntry semi = IteratorUtil.getNth(tokens.iterator(), 3); + TokenEntry semi = tokens.getTokens().get(3); assertEquals(";", getTokenImage(semi)); assertEquals(2, semi.getBeginLine()); assertEquals(2, semi.getBeginColumn()); @@ -50,15 +47,17 @@ private Tokens compareResult(AnyTokenizer tokenizer, String source, List SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(source)); Tokens tokens = new Tokens(); tokenizer.tokenize(code, tokens); - List tokenStrings = tokens.getTokens().stream() - .map(this::getTokenImage) - .collect(Collectors.toList()); + + List tokenStrings = new ArrayList<>(); + for (TokenEntry token : tokens.getTokens()) { + tokenStrings.add(getTokenImage(token)); + } assertEquals(expectedImages, tokenStrings); return tokens; } - private @NonNull String getTokenImage(TokenEntry t) { + private String getTokenImage(TokenEntry t) { return t.toString(); }