Skip to content

Commit

Permalink
Merge branch 'release/34.x'
Browse files Browse the repository at this point in the history 
* release/34.x:
  #5194 - Preferences when opening document in HTML editor
  No issue: Remove mentions of evaluation page in documentation since the page has been removed in v30
  #5191 - Support different types of line breaks
  • Loading branch information
@reckart
reckart committed Dec 1, 2024
2 parents 997af66 + b5e6f3f commit c28ea26
Show file tree
Hide file tree
Showing 15 changed files with 286 additions and 85 deletions.
53 changes: 40 additions & 13 deletions ...a/de/tudarmstadt/ukp/clarin/webanno/api/annotation/paging/LineOrientedPagingStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@
*/
package de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging;

import static org.apache.commons.lang3.StringUtils.splitPreserveAllTokens;
import static java.lang.String.join;
import static java.util.Arrays.asList;
import static java.util.regex.Pattern.compile;
import static java.util.stream.Collectors.joining;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

import org.apache.uima.cas.CAS;
import org.apache.wicket.Component;
Expand All @@ -37,22 +41,45 @@ public class LineOrientedPagingStrategy
{
private static final long serialVersionUID = -991967885210129525L;

static final String CR = "\r"; // carriage return (CR) (classic Mac)
static final String LF = "\n"; // line feed (LF) (Unix)
static final String CRLF = "\r\n"; // CRLF (Windows)
static final String NEL = "\u0085"; // Next Line (NEL)
static final String LINE_SEPARATOR = "\u2028"; // Line Separator
static final String PARAGRAPH_SEPARATOR = "\u2029"; // Paragraph Separator

// Mind that CRLF must come before CR and LF here so it matches as a unit!
static final List<String> LINE_SEPARATORS = asList(CRLF, CR, LF, NEL, LINE_SEPARATOR,
PARAGRAPH_SEPARATOR);

static final Pattern LINE_PATTERN = compile("[^" + join("", LINE_SEPARATORS) + "]+" //
+ "|" + join("|", LINE_SEPARATORS));
static final Pattern LINE_SPLITTER_PATTERN = compile(LINE_SEPARATORS.stream() //
.map(Pattern::quote) //
.collect(joining("|")));

@Override
public List<Unit> units(CAS aCas, int aFirstIndex, int aLastIndex)
{
// We need to preserve all tokens so we can add a +1 for the line breaks of empty lines.
String[] lines = splitPreserveAllTokens(aCas.getDocumentText(), '\n');
var text = aCas.getDocumentText();
var matcher = LINE_SPLITTER_PATTERN.matcher(text);

List<Unit> units = new ArrayList<>();
int beginOffset = 0;
for (int i = 0; i < Math.min(lines.length, aLastIndex); i++) {
var unitStart = 0;
var unitEnd = 0;
var index = 1;

if (i >= aFirstIndex) {
units.add(new Unit(i + 1, beginOffset, beginOffset + lines[i].length()));
}
var units = new ArrayList<Unit>();
while (matcher.find()) {
unitEnd = matcher.start();
units.add(new Unit(index, unitStart, unitEnd));
unitStart = matcher.end();
index++;
}

// The +1 below accounts for the line break which is not included in the token
beginOffset += lines[i].length() + 1;
if (unitStart < text.length()) {
if (!text.substring(unitStart).isBlank()) {
units.add(new Unit(index, unitStart, text.length()));
}
}

return units;
Expand All @@ -61,8 +88,8 @@ public List<Unit> units(CAS aCas, int aFirstIndex, int aLastIndex)
@Override
public Component createPositionLabel(String aId, IModel<AnnotatorState> aModel)
{
Label label = new Label(aId, () -> {
AnnotatorState state = aModel.getObject();
var label = new Label(aId, () -> {
var state = aModel.getObject();
return String.format("%d-%d / %d lines [doc %d / %d]", state.getFirstVisibleUnitIndex(),
state.getLastVisibleUnitIndex(), state.getUnitCount(),
state.getDocumentIndex() + 1, state.getNumberOfDocuments());
Expand Down
37 changes: 27 additions & 10 deletions .../de/tudarmstadt/ukp/clarin/webanno/api/annotation/paging/TokenWrappingPagingStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,22 @@
*/
package de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging;

import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging.LineOrientedPagingStrategy.LINE_SPLITTER_PATTERN;
import static java.lang.String.format;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.uima.cas.CAS;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.wicket.Component;
import org.apache.wicket.Page;
import org.apache.wicket.markup.html.basic.Label;
import org.apache.wicket.model.IModel;

import de.tudarmstadt.ukp.clarin.webanno.api.annotation.page.AnnotationPageBase;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;
import de.tudarmstadt.ukp.inception.rendering.editorstate.AnnotatorState;
import de.tudarmstadt.ukp.inception.rendering.paging.Unit;
import de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil;

public class TokenWrappingPagingStrategy
extends PagingStrategy_ImplBase
Expand All @@ -50,29 +49,29 @@ public TokenWrappingPagingStrategy(int aMaxLineLength)
@Override
public List<Unit> units(CAS aCas, int aFirstIndex, int aLastIndex)
{
Iterator<AnnotationFS> tokenIterator = WebAnnoCasUtil.selectTokens(aCas).iterator();
var tokenIterator = aCas.select(Token.class).iterator();

var units = new ArrayList<Unit>();

int currentUnitStart = 0;
int currentUnitEnd = 0;
while (tokenIterator.hasNext()) {
AnnotationFS currentToken = tokenIterator.next();
var currentToken = tokenIterator.next();

if (currentToken.getBegin() < currentUnitEnd) {
throw new IllegalStateException(format(
"Unable to render: Token at [%d-%d] illegally overlaps with previous token ending at [%d].",
currentToken.getBegin(), currentToken.getEnd(), currentUnitEnd));
}

// Add units for each of the lines in the gap
var gap = aCas.getDocumentText().substring(currentUnitEnd, currentToken.getBegin());
int gapStart = currentUnitEnd;
int lineBreakIndex = gap.indexOf("\n");
while (lineBreakIndex > -1) {
currentUnitEnd = gapStart + lineBreakIndex;
var matcher = LINE_SPLITTER_PATTERN.matcher(gap);
while (matcher.find()) {
currentUnitEnd = gapStart + matcher.start();
units.add(new Unit(units.size() + 1, currentUnitStart, currentUnitEnd));
currentUnitStart = currentUnitEnd + 1; // +1 because of the line break character
lineBreakIndex = gap.indexOf("\n", lineBreakIndex + 1);
currentUnitStart = gapStart + matcher.end();
}

var unitNonEmpty = (currentUnitEnd - currentUnitStart) > 0;
Expand All @@ -92,10 +91,28 @@ public List<Unit> units(CAS aCas, int aFirstIndex, int aLastIndex)
currentUnitEnd = currentToken.getEnd();
}

// Finish current unit
if (currentUnitEnd - currentUnitStart > 0) {
units.add(new Unit(units.size() + 1, currentUnitStart, currentUnitEnd));
currentUnitStart = -1;
}

// // Add any line breaks at the end of the document
// if (aCas.getDocumentText().length() - currentUnitEnd > 0) {
// var gap = aCas.getDocumentText().substring(currentUnitEnd,
// aCas.getDocumentText().length());
// int gapStart = currentUnitEnd;
// var matcher = LINE_SPLITTER_PATTERN.matcher(gap);
// while (matcher.find()) {
// currentUnitEnd = gapStart + matcher.start();
// if (currentUnitStart == -1) {
// currentUnitStart = currentUnitEnd;
// }
// units.add(new Unit(units.size() + 1, currentUnitStart, currentUnitEnd));
// currentUnitStart = gapStart + matcher.end();
// }
// }

return units;
}

Expand Down
101 changes: 101 additions & 0 deletions .../tudarmstadt/ukp/clarin/webanno/api/annotation/paging/LineOrientedPagingStrategyTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
* Licensed to the Technische Universität Darmstadt under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The Technische Universität Darmstadt
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License.
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging;

import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging.LineOrientedPagingStrategy.CR;
import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging.LineOrientedPagingStrategy.CRLF;
import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging.LineOrientedPagingStrategy.LF;
import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging.LineOrientedPagingStrategy.LINE_SEPARATOR;
import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging.LineOrientedPagingStrategy.NEL;
import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.paging.LineOrientedPagingStrategy.PARAGRAPH_SEPARATOR;
import static org.assertj.core.api.Assertions.assertThat;

import org.apache.uima.fit.factory.JCasFactory;
import org.junit.jupiter.api.Test;

class LineOrientedPagingStrategyTest
{
@Test
void testMixedLineBreaks() throws Exception
{
var sut = new LineOrientedPagingStrategy();

var jcas = JCasFactory.createJCas();
jcas.setDocumentText( //
"Line1" + CR + //
"Line2" + LF + //
"Line3" + CRLF + //
"Line4" + NEL + //
"Line5" + LINE_SEPARATOR + //
"Line6" + PARAGRAPH_SEPARATOR + //
"Line7");

assertThat(sut.units(jcas.getCas()))
.extracting(u -> jcas.getDocumentText().substring(u.getBegin(), u.getEnd()))
.containsExactly( //
"Line1", //
"Line2", //
"Line3", //
"Line4", //
"Line5", //
"Line6", //
"Line7");
}

@Test
void testConsecutiveLineBreaks() throws Exception
{
var sut = new LineOrientedPagingStrategy();

var jcas = JCasFactory.createJCas();
jcas.setDocumentText( //
"Line1" + CR + CR + //
"Line2" + LF + LF + //
"Line3" + CRLF + CRLF + //
"Line4" + NEL + NEL + //
"Line5" + LINE_SEPARATOR + LINE_SEPARATOR + //
"Line6" + PARAGRAPH_SEPARATOR + PARAGRAPH_SEPARATOR + //
"Line7");

assertThat(sut.units(jcas.getCas()))
.extracting(u -> jcas.getDocumentText().substring(u.getBegin(), u.getEnd()))
.containsExactly( //
"Line1", "", //
"Line2", "", //
"Line3", "", //
"Line4", "", //
"Line5", "", //
"Line6", "", //
"Line7");
}

@Test
void testEndingWithNewline() throws Exception
{
var sut = new LineOrientedPagingStrategy();

var jcas = JCasFactory.createJCas();
jcas.setDocumentText( //
"Line1" + CR + CR);

assertThat(sut.units(jcas.getCas()))
.extracting(u -> jcas.getDocumentText().substring(u.getBegin(), u.getEnd()))
.containsExactly( //
"Line1", "");
}
}
Loading

0 comments on commit c28ea26

Please sign in to comment.