Added documentation and better error messages.

docs/4.-Adding-New-Languages.md

@@ -510,6 +510,31 @@ protected File getTestFileLocation() {
 }
 ```
 
+## Testing token positions
+
+The precise position of a token can be relevant for the visualization in the report viewer. To make sure the token positions are extracted correctly language modules should include some test for that.
+
+Writing such tests can be done using a specific syntax in the test sources directly.
+Such a file can look like this:
+```java
+>class Test {
+>    int test;
+$    | J_VARDEF 8
+>}
+```
+
+Every line that is prefixed with '>' will be interpreted as a line of test source code.
+
+Every line starting with '$' contains information about one expected token. The token is expected in the first source line above this one.
+The '|' marks the column the token should be in. It is followed by one space, then the name of the token (The name of the enum constant).
+Finally separated with another space is the length of the token.
+A single file may contain any number of tokens.
+The test will verify that at least one token with those exact properties exist.
+
+These test files have to be added in the TestDataCollector. Put all test files in a single directory and specify it though collector.addTokenPositionTests("<directory>").
+If the directory is in the default location for test files, a relative path is enough, otherwise a full path has to be specified.
+
+
 # Adding code highlighting to the report-viewer
 To ensure your language gets properly registered and its code is correctly highlighted in the report-viewer:
 1) Add your language to the `ParserLanguage` enum in 'src/model/Language.ts'. As the value for the entry use its frontend name.

language-testutils/src/test/java/de/jplag/testutils/LanguageModuleTest.java

@@ -176,7 +176,7 @@ final List<TestDataCollector.TokenListTest> testTokensContainedData() {
     final void testTokenSequence(TestDataCollector.TokenListTest test) throws ParsingException, IOException {
         List<TokenType> actual = extractTokenTypes(test.data());
         List<TokenType> expected = new ArrayList<>(test.tokens());
-        if (expected.get(expected.size() - 1) != SharedTokenType.FILE_END) {
+        if (expected.getLast() != SharedTokenType.FILE_END) {
             expected.add(SharedTokenType.FILE_END);
         }
         assertTokensMatch(expected, actual, "Extracted token from " + test.data().describeTestSource() + " does not match expected sequence.");
@@ -198,6 +198,12 @@ final List<TestDataCollector.TokenListTest> testTokenSequenceData() {
         return ignoreEmptyTestType(this.collector.getTokenSequenceTest());
     }
 
+    /**
+     * Tests if the tokens specified for the token position tests are present in the sources
+     * @param testData The specifications of the expected tokens and the test source
+     * @throws ParsingException If the parsing fails
+     * @throws IOException If IO operations fail. If this happens, that should be unrelated to the test itself.
+     */
     @ParameterizedTest
     @MethodSource("getTokenPositionTestData")
     @DisplayName("Tests if the extracted tokens contain the tokens specified in the test files.")
@@ -206,7 +212,8 @@ final void testTokenPositions(TokenPositionTestData testData) throws ParsingExce
         List<TokenPositionTestData.TokenData> failedTokens = new ArrayList<>();
 
         for (TokenPositionTestData.TokenData expectedToken : testData.getExpectedTokens()) {
-            TokenType expectedType = this.languageTokens.stream().filter(type -> type.toString().equals(expectedToken.typeName())).findFirst().get();
+            TokenType expectedType = this.languageTokens.stream().filter(type -> type.toString().equals(expectedToken.typeName())).findFirst()
+                    .orElseThrow(() -> new IOException(String.format("The token type %s does not exist.", expectedToken.typeName())));
 
             if (extractedTokens.stream().noneMatch(token -> token.getType() == expectedType && token.getLine() == expectedToken.line()
                     && token.getColumn() == expectedToken.col() && token.getLength() == expectedToken.length())) {
@@ -221,6 +228,9 @@ final void testTokenPositions(TokenPositionTestData testData) throws ParsingExce
         }
     }
 
+    /**
+     * @return All token positions tests that are configured
+     */
     final List<TokenPositionTestData> getTokenPositionTestData() {
         return ignoreEmptyTestType(this.collector.getTokenPositionTestData());
     }
@@ -260,8 +270,7 @@ final void testMonotoneTokenOrder(TestData data) throws ParsingException, IOExce
     final void testTokenSequencesEndsWithFileEnd(TestData data) throws ParsingException, IOException {
         List<Token> tokens = parseTokens(data);
 
-        assertEquals(SharedTokenType.FILE_END, tokens.get(tokens.size() - 1).getType(),
-                "Last token in " + data.describeTestSource() + " is not file end.");
+        assertEquals(SharedTokenType.FILE_END, tokens.getLast().getType(), "Last token in " + data.describeTestSource() + " is not file end.");
     }
 
     /**

language-testutils/src/test/java/de/jplag/testutils/TmpFileHolder.java

@@ -4,9 +4,15 @@
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * Stores all temporary files that are created for a {@link LanguageModuleTest} and provides the option to delete them
+ */
 public class TmpFileHolder {
     public static List<File> tmpFiles = new ArrayList<>();
 
+    /**
+     * Deletes all temporary files that have been created up to this point
+     */
     public static void deleteTmpFiles() {
         tmpFiles.forEach(File::delete);
         tmpFiles.clear();

language-testutils/src/test/java/de/jplag/testutils/datacollector/TokenPositionTestData.java

@@ -12,12 +12,20 @@
 import de.jplag.testutils.TmpFileHolder;
 import de.jplag.util.FileUtils;
 
+/**
+ * Test sources with token information Reads token position test specifications form a file and provides the token
+ * information for tests. The sources cna be used as regular test sources.
+ */
 public class TokenPositionTestData implements TestData {
     private final List<String> sourceLines;
     private final List<TokenData> expectedTokens;
 
     private final String descriptor;
 
+    /**
+     * @param testFile The file containing the test specifications
+     * @throws IOException If the file cannot be read
+     */
     public TokenPositionTestData(File testFile) throws IOException {
         this.sourceLines = new ArrayList<>();
         this.expectedTokens = new ArrayList<>();
@@ -65,10 +73,20 @@ public String describeTestSource() {
         return this.descriptor;
     }
 
+    /**
+     * @return A list of the expected tokens for this test source
+     */
     public List<TokenData> getExpectedTokens() {
         return expectedTokens;
     }
 
+    /**
+     * Information about a single token
+     * @param typeName The name of the token type
+     * @param line The line the token is in
+     * @param col The column the token is in
+     * @param length The length of the token
+     */
     public record TokenData(String typeName, int line, int col, int length) {
     }
 }