diff --git a/Jenkinsfile b/Jenkinsfile index f3cd94ad292..3a2013a489f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ config = [ agentLabel: '', maven: 'Maven 3', - jdk: 'Zulu 11', + jdk: 'Zulu 17', extraMavenArguments: '-U -Ddkpro.core.testCachePath="${WORKSPACE}/cache/dkpro-core-datasets" -Dmaven.artifact.threads=15', wipeWorkspaceBeforeBuild: true, wipeWorkspaceAfterBuild: true @@ -28,7 +28,7 @@ pipeline { } agent { - label agentLabel + label params.agentLabel } tools { diff --git a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningService.java b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningService.java index 4cd7e362f4c..2549de5f2a4 100644 --- a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningService.java +++ b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningService.java @@ -25,7 +25,6 @@ import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.clarin.webanno.security.model.User; import de.tudarmstadt.ukp.inception.active.learning.ActiveLearningServiceImpl.ActiveLearningUserState; -import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecord; import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordType; import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion; import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup; @@ -44,15 +43,6 @@ public interface ActiveLearningService */ List> getSuggestions(User aDataOwner, AnnotationLayer aLayer); - /** - * @param aRecord - * record to check - * @return if the suggestions from which the given record was created (or an equivalent one) is - * visible to the user. This is useful to check if the suggestion can be highlighted - * when clicking on a history record. - */ - boolean isSuggestionVisible(LearningRecord aRecord); - /** * @return if the are any records of type {@link LearningRecordType#SKIPPED} in the history of * the given layer for the given user. @@ -64,8 +54,8 @@ public interface ActiveLearningService */ boolean hasSkippedSuggestions(String aSessionOwner, User aDataOwner, AnnotationLayer aLayer); - void hideRejectedOrSkippedAnnotations(String aSessionOwner, User aDataOwner, AnnotationLayer aLayer, - boolean aFilterSkippedRecommendation, + void hideRejectedOrSkippedAnnotations(String aSessionOwner, User aDataOwner, + AnnotationLayer aLayer, boolean aFilterSkippedRecommendation, List> aSuggestionGroups); Optional> generateNextSuggestion(String aSessionOwner, User aDataOwner, diff --git a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningServiceImpl.java b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningServiceImpl.java index 5a9dffbc25f..df12932ac33 100644 --- a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningServiceImpl.java +++ b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/ActiveLearningServiceImpl.java @@ -27,8 +27,8 @@ import java.io.IOException; import java.io.Serializable; +import java.lang.invoke.MethodHandles; import java.util.List; -import java.util.Map; import java.util.Optional; import org.slf4j.Logger; @@ -49,10 +49,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.LearningRecordService; import de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService; import de.tudarmstadt.ukp.inception.recommendation.api.model.AnnotationSuggestion; -import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecord; -import de.tudarmstadt.ukp.inception.recommendation.api.model.Predictions; import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion; -import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionDocumentGroup; import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup; import de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionGroup.Delta; import de.tudarmstadt.ukp.inception.schema.AnnotationSchemaService; @@ -68,7 +65,7 @@ public class ActiveLearningServiceImpl implements ActiveLearningService { - private final Logger log = LoggerFactory.getLogger(getClass()); + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final ApplicationEventPublisher applicationEventPublisher; private final DocumentService documentService; @@ -97,39 +94,20 @@ public ActiveLearningServiceImpl(DocumentService aDocumentService, @Override public List> getSuggestions(User aUser, AnnotationLayer aLayer) { - Predictions predictions = recommendationService.getPredictions(aUser, aLayer.getProject()); + var predictions = recommendationService.getPredictions(aUser, aLayer.getProject()); if (predictions == null) { return emptyList(); } - Map> recommendationsMap = predictions - .getPredictionsForWholeProject(SpanSuggestion.class, aLayer, documentService); + var recommendationsMap = predictions.getPredictionsForWholeProject(SpanSuggestion.class, + aLayer, documentService); return recommendationsMap.values().stream() // .flatMap(docMap -> docMap.stream()) // .collect(toList()); } - @Override - public boolean isSuggestionVisible(LearningRecord aRecord) - { - var aSessionOwner = userService.get(aRecord.getUser()); - var suggestionGroups = getSuggestions(aSessionOwner, aRecord.getLayer()); - for (var suggestionGroup : suggestionGroups) { - if (suggestionGroup.stream().anyMatch(suggestion -> suggestion.getDocumentName() - .equals(aRecord.getSourceDocument().getName()) - && suggestion.getFeature().equals(aRecord.getAnnotationFeature().getName()) - && suggestion.labelEquals(aRecord.getAnnotation()) - && suggestion.getBegin() == aRecord.getOffsetBegin() - && suggestion.getEnd() == aRecord.getOffsetEnd() // - && suggestion.isVisible())) { - return true; - } - } - return false; - } - @Override public boolean hasSkippedSuggestions(String aSessionOwner, User aDataOwner, AnnotationLayer aLayer) @@ -174,27 +152,31 @@ public Optional> generateNextSuggestion(String aSessionOwn long startTimer = System.currentTimeMillis(); var suggestionGroups = alState.getSuggestions(); long getRecommendationsFromRecommendationService = System.currentTimeMillis(); - log.trace("Getting recommendations from recommender system took {} ms.", + LOG.trace("Getting recommendations from recommender system took {} ms.", (getRecommendationsFromRecommendationService - startTimer)); - // remove duplicate recommendations - suggestionGroups = suggestionGroups.stream() // - .map(it -> removeDuplicateRecommendations(it)) // - .collect(toList()); - long removeDuplicateRecommendation = System.currentTimeMillis(); - log.trace("Removing duplicate recommendations took {} ms.", - (removeDuplicateRecommendation - getRecommendationsFromRecommendationService)); - // hide rejected recommendations hideRejectedOrSkippedAnnotations(aSessionOwner, aDataOwner, alState.getLayer(), true, suggestionGroups); long removeRejectedSkippedRecommendation = System.currentTimeMillis(); - log.trace("Removing rejected or skipped ones took {} ms.", - (removeRejectedSkippedRecommendation - removeDuplicateRecommendation)); + LOG.trace("Hiding rejected or skipped ones took {} ms.", + (removeRejectedSkippedRecommendation + - getRecommendationsFromRecommendationService)); + + // remove duplicate recommendations + suggestionGroups = suggestionGroups.stream() // + .map(it -> removeDuplicatesAndHiddenSuggestions(it)) // + .filter(it -> !it.isEmpty()) // + .collect(toList()); + long removeDuplicateRecommendation = System.currentTimeMillis(); + LOG.trace("Removing duplicate recommendations took {} ms.", + (removeDuplicateRecommendation - removeRejectedSkippedRecommendation)); var pref = recommendationService.getPreferences(aDataOwner, alState.getLayer().getProject()); - return alState.getStrategy().generateNextSuggestion(pref, suggestionGroups); + var nextSuggestion = alState.getStrategy().generateNextSuggestion(pref, suggestionGroups); + assert nextSuggestion.get().getFirst().isVisible() : "Generated suggestion must be visible"; + return nextSuggestion; } @Override @@ -296,35 +278,40 @@ public void skipSpanSuggestion(String aSessionOwner, User aDataOwner, Annotation alternativeSuggestions)); } - private static SuggestionGroup removeDuplicateRecommendations( - SuggestionGroup unmodifiedRecommendationList) + private static SuggestionGroup removeDuplicatesAndHiddenSuggestions( + SuggestionGroup aSuggestionGroup) { - SuggestionGroup cleanRecommendationList = new SuggestionGroup<>(); + var cleanSuggestionGroup = new SuggestionGroup(); - unmodifiedRecommendationList.forEach(recommendationItem -> { - if (!isAlreadyInCleanList(cleanRecommendationList, recommendationItem)) { - cleanRecommendationList.add(recommendationItem); + aSuggestionGroup.forEach(suggestion -> { + if (!suggestion.isVisible()) { + return; + } + + if (!isAlreadyInCleanList(cleanSuggestionGroup, suggestion)) { + cleanSuggestionGroup.add(suggestion); } }); - return cleanRecommendationList; + return cleanSuggestionGroup; } private static boolean isAlreadyInCleanList( SuggestionGroup cleanRecommendationList, AnnotationSuggestion recommendationItem) { - String source = recommendationItem.getRecommenderName(); - String annotation = recommendationItem.getLabel(); - String documentName = recommendationItem.getDocumentName(); + var source = recommendationItem.getRecommenderName(); + var annotation = recommendationItem.getLabel(); + var documentName = recommendationItem.getDocumentName(); - for (AnnotationSuggestion existingRecommendation : cleanRecommendationList) { - boolean areLabelsEqual = existingRecommendation.labelEquals(annotation); + for (var existingRecommendation : cleanRecommendationList) { + var areLabelsEqual = existingRecommendation.labelEquals(annotation); if (existingRecommendation.getRecommenderName().equals(source) && areLabelsEqual && existingRecommendation.getDocumentName().equals(documentName)) { return true; } } + return false; } diff --git a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/sidebar/ActiveLearningSidebar.java b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/sidebar/ActiveLearningSidebar.java index c5b026063d0..60f8a33aea1 100644 --- a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/sidebar/ActiveLearningSidebar.java +++ b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/sidebar/ActiveLearningSidebar.java @@ -75,7 +75,6 @@ import de.tudarmstadt.ukp.clarin.webanno.model.ReorderableTag; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.clarin.webanno.security.UserDao; -import de.tudarmstadt.ukp.clarin.webanno.security.model.User; import de.tudarmstadt.ukp.clarin.webanno.support.bootstrap.BootstrapModalDialog; import de.tudarmstadt.ukp.clarin.webanno.support.lambda.LambdaAjaxButton; import de.tudarmstadt.ukp.clarin.webanno.support.lambda.LambdaAjaxLink; @@ -302,10 +301,10 @@ private List listLayersWithRecommenders() private void actionStartSession(AjaxRequestTarget aTarget, Form form) { - ActiveLearningUserState alState = alStateModel.getObject(); - AnnotatorState state = getModelObject(); - String userName = state.getUser().getUsername(); - Project project = state.getProject(); + var alState = alStateModel.getObject(); + var state = getModelObject(); + var userName = state.getUser().getUsername(); + var project = state.getProject(); recommendationService.setPredictForAllDocuments(userName, project, true); recommendationService.triggerPrediction(userName, "ActionStartActiveLearningSession", @@ -333,6 +332,7 @@ private void actionStartSession(AjaxRequestTarget aTarget, Form form) */ private void requestClearningSelectionAndJumpingToSuggestion() { + LOG.trace("Requesting clearing and jumping"); RequestCycle.get().setMetaData(ClearSelectionAndJumpToSuggestionKey.INSTANCE, true); } @@ -365,7 +365,7 @@ private void actionStopSession(AjaxRequestTarget aTarget) private void setActiveLearningHighlight(SpanSuggestion aSuggestion) { - assert aSuggestion.isVisible(); + assert aSuggestion.isVisible() : "Cannot highlight hidden suggestions"; if (protectHighlight) { LOG.trace("Active learning sidebar not updating protected highlights"); @@ -519,12 +519,18 @@ private LambdaAjaxLink createJumpToSuggestionLink() private void actionJumpToSuggestion(AjaxRequestTarget aTarget) throws IOException { - ActiveLearningUserState alState = alStateModel.getObject(); - SpanSuggestion suggestion = alState.getSuggestion().get(); + var alState = alStateModel.getObject(); + var suggestion = alState.getSuggestion().get(); + + if (!suggestion.isVisible()) { + error("Cannot jump to hidden suggestion"); + aTarget.addChildren(getPage(), IFeedback.class); + return; + } if (LOG.isDebugEnabled()) { LOG.debug("Active suggestion: {}", suggestion); - Optional updatedSuggestion = getMatchingSuggestion(activeLearningService + var updatedSuggestion = getMatchingSuggestion(activeLearningService .getSuggestions(getModelObject().getUser(), alState.getLayer()), suggestion) .stream().findFirst(); updatedSuggestion.ifPresent(s -> LOG.debug("Update suggestion: {}", s)); @@ -732,13 +738,14 @@ private void moveToNextSuggestion(AjaxRequestTarget aTarget) var sessionOwner = userService.getCurrentUser(); // Generate the next recommendation but remember the current one - Optional prevSuggestion = alState.getSuggestion(); - alState.setCurrentDifference(activeLearningService - .generateNextSuggestion(sessionOwner.getUsername(), dataOwner, alState)); + var currentSuggestion = alState.getSuggestion(); + var nextSuggestion = activeLearningService + .generateNextSuggestion(sessionOwner.getUsername(), dataOwner, alState); + alState.setCurrentDifference(nextSuggestion); // If there is no new suggestion, nothing left to do here if (!alState.getSuggestion().isPresent()) { - if (prevSuggestion.isPresent()) { + if (currentSuggestion.isPresent()) { infoOnce(aTarget, "There are no more recommendations right now."); } @@ -749,10 +756,11 @@ private void moveToNextSuggestion(AjaxRequestTarget aTarget) } // If the active suggestion has changed, inform the user - if (prevSuggestion.isPresent() - && !alState.getSuggestion().get().equals(prevSuggestion.get())) { + if (currentSuggestion.isPresent() + && !alState.getSuggestion().get().equals(currentSuggestion.get())) { // infoOnce(aTarget, "Active learning has moved to next best suggestion."); - LOG.trace("Moving from {} to {}", prevSuggestion.get(), alState.getSuggestion().get()); + LOG.trace("Moving from {} to {}", currentSuggestion.get(), + alState.getSuggestion().get()); } // If there is a suggestion, open it in the sidebar and take the main editor to its location @@ -778,7 +786,7 @@ private void moveToNextSuggestion(AjaxRequestTarget aTarget) private void clearSelectedAnnotationAndJumpToSuggestion(AjaxRequestTarget aTarget) { - ActiveLearningUserState alState = alStateModel.getObject(); + var alState = alStateModel.getObject(); if (!alState.getSuggestion().isPresent()) { return; } @@ -787,11 +795,11 @@ private void clearSelectedAnnotationAndJumpToSuggestion(AjaxRequestTarget aTarge // I.e. we must not make the editor/feature details jump to the next suggestion but rather // keep the view and selected annotation that the user has chosen to provide the opportunity // to the user to continue editing on it - SpanSuggestion suggestion = alState.getSuggestion().get(); - AnnotatorState state = getModelObject(); - Project project = state.getProject(); - User user = state.getUser(); - SourceDocument sourceDocument = documentService.getSourceDocument(project, + var suggestion = alState.getSuggestion().get(); + var state = getModelObject(); + var project = state.getProject(); + var user = state.getUser(); + var sourceDocument = documentService.getSourceDocument(project, suggestion.getDocumentName()); LOG.trace("Jumping to {}", suggestion); @@ -829,12 +837,12 @@ private void loadSuggestionInActiveLearningSidebar(AjaxRequestTarget aTarget, // Obtain some left and right context of the active suggestion while we have easy // access to the document which contains the current suggestion try { - CAS cas = documentService.readAnnotationCas(sourceDocument, + var cas = documentService.readAnnotationCas(sourceDocument, getModelObject().getUser().getUsername(), AUTO_CAS_UPGRADE, SHARED_READ_ONLY_ACCESS); - String text = cas.getDocumentText(); + var text = cas.getDocumentText(); - ActiveLearningUserState alState = alStateModel.getObject(); + var alState = alStateModel.getObject(); alState.setLeftContext( text.substring(Math.max(0, suggestion.getBegin() - 20), suggestion.getBegin())); alState.setRightContext(text.substring(suggestion.getEnd(), @@ -872,8 +880,8 @@ private ListView createLearningHistoryListView() @Override protected void populateItem(ListItem item) { - LearningRecord rec = item.getModelObject(); - AnnotationFeature recAnnotationFeature = rec.getAnnotationFeature(); + var rec = item.getModelObject(); + var recAnnotationFeature = rec.getAnnotationFeature(); String recFeatureValue; if (recAnnotationFeature != null) { FeatureSupport featureSupport = featureSupportRegistry @@ -885,7 +893,7 @@ protected void populateItem(ListItem item) recFeatureValue = rec.getAnnotation(); } - LambdaAjaxLink textLink = new LambdaAjaxLink(CID_JUMP_TO_ANNOTATION, + var textLink = new LambdaAjaxLink(CID_JUMP_TO_ANNOTATION, _target -> actionSelectHistoryItem(_target, item.getModelObject())); textLink.setBody(rec::getTokenText); item.add(textLink); @@ -923,10 +931,10 @@ private void actionSelectHistoryItem(AjaxRequestTarget aTarget, LearningRecord a // Since we have switched documents above (if it was necessary), the editor CAS should // now point to the correct one - CAS cas = getCasProvider().get(); + var cas = getCasProvider().get(); // ... if a matching annotation exists, highlight the annotaiton - Optional annotation = getMatchingAnnotation(cas, aRecord); + var annotation = getMatchingAnnotation(cas, aRecord); if (annotation.isPresent()) { setActiveLearningHighlight(aRecord.getSourceDocument(), annotation.get()); @@ -976,7 +984,8 @@ private List getMatchingSuggestion( && (aFeature == null || aFeature.equals(group.getFeature())) && (aBegin == -1 || aBegin == ((Offset) group.getPosition()).getBegin()) && (aEnd == -1 || aEnd == ((Offset) group.getPosition()).getEnd())) - .flatMap(group -> group.stream()) + .flatMap(group -> group.stream()) // + .filter(suggestion -> suggestion.isVisible()) // .filter(suggestion -> aLabel == null || aLabel.equals(suggestion.getLabel())) .collect(toList()); } @@ -1360,10 +1369,10 @@ private void refreshAvailableSuggestions() { LOG.trace("refreshAvailableSuggestions()"); - AnnotatorState state = getModelObject(); - ActiveLearningUserState alState = alStateModel.getObject(); - alState.setSuggestions( - activeLearningService.getSuggestions(state.getUser(), alState.getLayer())); + var state = getModelObject(); + var alState = alStateModel.getObject(); + var suggestions = activeLearningService.getSuggestions(state.getUser(), alState.getLayer()); + alState.setSuggestions(suggestions); } @OnEvent @@ -1420,11 +1429,11 @@ private void refreshCurrentSuggestionOrMoveToNextSuggestion(AjaxRequestTarget aT // is still relevant - if yes, we need to replace it with its current counterpart since. // if no counterpart exists in the current suggestions, then we need to load a // suggestion from the current list. - ActiveLearningUserState alState = alStateModel.getObject(); - SpanSuggestion activeSuggestion = alState.getSuggestion().get(); + var alState = alStateModel.getObject(); + var activeSuggestion = alState.getSuggestion().get(); // Find the groups which matches the active recommendation - Optional updatedSuggestion = getMatchingSuggestion(alState.getSuggestions(), - activeSuggestion).stream().findFirst(); + var updatedSuggestion = getMatchingSuggestion(alState.getSuggestions(), activeSuggestion) + .stream().findFirst(); if (updatedSuggestion.isEmpty()) { moveToNextSuggestion(aTarget); @@ -1432,8 +1441,7 @@ private void refreshCurrentSuggestionOrMoveToNextSuggestion(AjaxRequestTarget aT } LOG.debug("Replacing outdated suggestion {} with new suggestion {}", - alState.getCurrentDifference().get().getFirst().getId(), - updatedSuggestion.get().getId()); + alState.getCurrentDifference().get().getFirst(), updatedSuggestion.get()); // Update the highlight if (alState.getSuggestion().get().getVID().equals(highlightVID)) { diff --git a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/strategy/UncertaintySamplingStrategy.java b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/strategy/UncertaintySamplingStrategy.java index a35a5f092e0..79eec16243e 100644 --- a/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/strategy/UncertaintySamplingStrategy.java +++ b/inception/inception-active-learning/src/main/java/de/tudarmstadt/ukp/inception/active/learning/strategy/UncertaintySamplingStrategy.java @@ -34,9 +34,9 @@ public class UncertaintySamplingStrategy @Override public Optional> generateNextSuggestion(Preferences aPreferences, - List> suggestions) + List> aSuggestions) { - return suggestions.stream() + return aSuggestions.stream() // Fetch the top deltas per recommender .flatMap(group -> group.getTopDeltas(aPreferences).values().stream()) // ... sort them in ascending order (smallest delta first) diff --git a/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/doccat/OpenNlpDoccatRecommender.java b/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/doccat/OpenNlpDoccatRecommender.java index f246427a4f3..879ec7cac49 100644 --- a/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/doccat/OpenNlpDoccatRecommender.java +++ b/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/doccat/OpenNlpDoccatRecommender.java @@ -19,6 +19,7 @@ import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil.selectOverlapping; import static de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult.toEvaluationResult; +import static de.tudarmstadt.ukp.inception.rendering.model.Range.rangeCoveringAnnotations; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.uima.fit.util.CasUtil.getType; import static org.apache.uima.fit.util.CasUtil.indexCovered; @@ -72,6 +73,9 @@ public class OpenNlpDoccatRecommender private static final Class SAMPLE_UNIT = Sentence.class; private static final Class DATAPOINT_UNIT = Sentence.class; + private static final int MIN_TRAINING_SET_SIZE = 2; + private static final int MIN_TEST_SET_SIZE = 2; + private final OpenNlpDoccatRecommenderTraits traits; public OpenNlpDoccatRecommender(Recommender aRecommender, @@ -91,7 +95,7 @@ public boolean isReadyForPrediction(RecommenderContext aContext) @Override public void train(RecommenderContext aContext, List aCasses) throws RecommendationException { - List docSamples = extractSamples(aCasses); + var docSamples = extractSamples(aCasses); if (docSamples.size() < 2) { aContext.warn("Not enough training data: [%d] items", docSamples.size()); @@ -109,10 +113,10 @@ public void train(RecommenderContext aContext, List aCasses) throws Recomme // OpenNLP int beamSize = Math.max(maxRecommendations, NameFinderME.DEFAULT_BEAM_SIZE); - TrainingParameters params = traits.getParameters(); + var params = traits.getParameters(); params.put(BeamSearch.BEAM_SIZE_PARAMETER, Integer.toString(beamSize)); - DoccatModel model = train(docSamples, params); + var model = train(docSamples, params); aContext.put(KEY_MODEL, model); } @@ -127,43 +131,42 @@ public TrainingCapability getTrainingCapability() public Range predict(RecommenderContext aContext, CAS aCas, int aBegin, int aEnd) throws RecommendationException { - DoccatModel model = aContext.get(KEY_MODEL).orElseThrow( + var model = aContext.get(KEY_MODEL).orElseThrow( () -> new RecommendationException("Key [" + KEY_MODEL + "] not found in context")); - DocumentCategorizerME finder = new DocumentCategorizerME(model); + var finder = new DocumentCategorizerME(model); - Type sampleUnitType = getType(aCas, SAMPLE_UNIT); - Type predictedType = getPredictedType(aCas); - Type tokenType = getType(aCas, Token.class); - Feature scoreFeature = getScoreFeature(aCas); - Feature predictedFeature = getPredictedFeature(aCas); - Feature isPredictionFeature = getIsPredictionFeature(aCas); + var sampleUnitType = getType(aCas, SAMPLE_UNIT); + var predictedType = getPredictedType(aCas); + var tokenType = getType(aCas, Token.class); + var scoreFeature = getScoreFeature(aCas); + var predictedFeature = getPredictedFeature(aCas); + var isPredictionFeature = getIsPredictionFeature(aCas); var units = selectOverlapping(aCas, sampleUnitType, aBegin, aEnd); - int predictionCount = 0; - for (AnnotationFS sampleUnit : units) { + var predictionCount = 0; + for (var unit : units) { if (predictionCount >= traits.getPredictionLimit()) { break; } predictionCount++; - List tokenAnnotations = selectCovered(tokenType, sampleUnit); - String[] tokens = tokenAnnotations.stream() // + var tokenAnnotations = selectCovered(tokenType, unit); + var tokens = tokenAnnotations.stream() // .map(AnnotationFS::getCoveredText) // .toArray(String[]::new); - double[] outcome = finder.categorize(tokens); - String label = finder.getBestCategory(outcome); + var outcome = finder.categorize(tokens); + var label = finder.getBestCategory(outcome); - AnnotationFS annotation = aCas.createAnnotation(predictedType, sampleUnit.getBegin(), - sampleUnit.getEnd()); + var annotation = aCas.createAnnotation(predictedType, unit.getBegin(), unit.getEnd()); annotation.setStringValue(predictedFeature, label); annotation.setDoubleValue(scoreFeature, NumberUtils.max(outcome)); annotation.setBooleanValue(isPredictionFeature, true); aCas.addFsToIndexes(annotation); } - return new Range(units); + return rangeCoveringAnnotations(units); } @Override @@ -176,9 +179,9 @@ public int estimateSampleCount(List aCasses) public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) throws RecommendationException { - List data = extractSamples(aCasses); - List trainingSet = new ArrayList<>(); - List testSet = new ArrayList<>(); + var data = extractSamples(aCasses); + var trainingSet = new ArrayList(); + var testSet = new ArrayList(); for (DocumentSample nameSample : data) { switch (aDataSplitter.getTargetSet(nameSample)) { @@ -194,39 +197,33 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) } } - int testSetSize = testSet.size(); - int trainingSetSize = trainingSet.size(); - double overallTrainingSize = data.size() - testSetSize; - double trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0; - - final int minTrainingSetSize = 2; - final int minTestSetSize = 2; - if (trainingSetSize < minTrainingSetSize || testSetSize < minTestSetSize) { - if ((getRecommender().getThreshold() <= 0.0d)) { - return new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), - SAMPLE_UNIT.getSimpleName()); - } + var testSetSize = testSet.size(); + var trainingSetSize = trainingSet.size(); + var overallTrainingSize = data.size() - testSetSize; + var trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0; - String info = String.format( - "Not enough evaluation data: training set [%s] items, test set [%s] of total [%s]", - trainingSetSize, testSetSize, data.size()); - LOG.info(info); + if (trainingSetSize < MIN_TRAINING_SET_SIZE || testSetSize < MIN_TEST_SET_SIZE) { + String msg = String.format( + "Not enough evaluation data: training set size [%d] (min. %d), test set size [%d] (min. %d) of total [%d] (min. %d)", + trainingSetSize, MIN_TRAINING_SET_SIZE, testSetSize, MIN_TEST_SET_SIZE, + data.size(), (MIN_TRAINING_SET_SIZE + MIN_TEST_SET_SIZE)); + LOG.info(msg); - EvaluationResult result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), + var result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), SAMPLE_UNIT.getSimpleName(), trainingSetSize, testSetSize, trainRatio); result.setEvaluationSkipped(true); - result.setErrorMsg(info); + result.setErrorMsg(msg); return result; } if (trainingSet.stream().map(DocumentSample::getCategory).distinct().count() <= 1) { - String info = String.format("Training data requires at least two different labels"); - LOG.info(info); + var msg = String.format("Training data requires at least two different labels"); + LOG.info(msg); - EvaluationResult result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), + var result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), SAMPLE_UNIT.getSimpleName(), trainingSetSize, testSetSize, trainRatio); result.setEvaluationSkipped(true); - result.setErrorMsg(info); + result.setErrorMsg(msg); return result; } @@ -234,11 +231,11 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) trainingSet.size(), testSet.size()); // Train model - DoccatModel model = train(trainingSet, traits.getParameters()); - DocumentCategorizerME doccat = new DocumentCategorizerME(model); + var model = train(trainingSet, traits.getParameters()); + var doccat = new DocumentCategorizerME(model); // Evaluate - EvaluationResult result = testSet.stream() + var result = testSet.stream() .map(sample -> new LabelPair(sample.getCategory(), doccat.getBestCategory(doccat.categorize(sample.getText())))) .collect(toEvaluationResult(DATAPOINT_UNIT.getSimpleName(), @@ -250,7 +247,7 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) private List extractSamples(List aCasses) { - List samples = new ArrayList<>(); + var samples = new ArrayList(); casses: for (CAS cas : aCasses) { Type sampleUnitType = getType(cas, SAMPLE_UNIT); Type tokenType = getType(cas, Token.class); diff --git a/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/ner/OpenNlpNerRecommender.java b/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/ner/OpenNlpNerRecommender.java index b1f9dc4a060..31f24694533 100644 --- a/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/ner/OpenNlpNerRecommender.java +++ b/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/ner/OpenNlpNerRecommender.java @@ -19,6 +19,7 @@ import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil.selectOverlapping; import static de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult.toEvaluationResult; +import static de.tudarmstadt.ukp.inception.rendering.model.Range.rangeCoveringAnnotations; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.uima.fit.util.CasUtil.getType; @@ -73,6 +74,9 @@ public class OpenNlpNerRecommender private static final Class SAMPLE_UNIT = Sentence.class; private static final Class DATAPOINT_UNIT = Token.class; + private static final int MIN_TRAINING_SET_SIZE = 2; + private static final int MIN_TEST_SET_SIZE = 2; + private final OpenNlpNerRecommenderTraits traits; public OpenNlpNerRecommender(Recommender aRecommender, OpenNlpNerRecommenderTraits aTraits) @@ -121,40 +125,41 @@ public TrainingCapability getTrainingCapability() public Range predict(RecommenderContext aContext, CAS aCas, int aBegin, int aEnd) throws RecommendationException { - TokenNameFinderModel model = aContext.get(KEY_MODEL).orElseThrow( + var model = aContext.get(KEY_MODEL).orElseThrow( () -> new RecommendationException("Key [" + KEY_MODEL + "] not found in context")); - NameFinderME finder = new NameFinderME(model); + var finder = new NameFinderME(model); - Type sampleUnitType = getType(aCas, SAMPLE_UNIT); - Type tokenType = getType(aCas, Token.class); - Type predictedType = getPredictedType(aCas); + var sampleUnitType = getType(aCas, SAMPLE_UNIT); + var tokenType = getType(aCas, Token.class); + var predictedType = getPredictedType(aCas); - Feature predictedFeature = getPredictedFeature(aCas); - Feature isPredictionFeature = getIsPredictionFeature(aCas); - Feature scoreFeature = getScoreFeature(aCas); + var predictedFeature = getPredictedFeature(aCas); + var isPredictionFeature = getIsPredictionFeature(aCas); + var scoreFeature = getScoreFeature(aCas); var units = selectOverlapping(aCas, sampleUnitType, aBegin, aEnd); - int predictionCount = 0; - for (AnnotationFS sampleUnit : units) { + var predictionCount = 0; + + for (var unit : units) { if (predictionCount >= traits.getPredictionLimit()) { break; } predictionCount++; - List tokenAnnotations = selectCovered(tokenType, sampleUnit); - String[] tokens = tokenAnnotations.stream() // + var tokenAnnotations = selectCovered(tokenType, unit); + var tokens = tokenAnnotations.stream() // .map(AnnotationFS::getCoveredText) // .toArray(String[]::new); - for (Span prediction : finder.find(tokens)) { - String label = prediction.getType(); + for (var prediction : finder.find(tokens)) { + var label = prediction.getType(); if (NameSample.DEFAULT_TYPE.equals(label)) { continue; } int begin = tokenAnnotations.get(prediction.getStart()).getBegin(); int end = tokenAnnotations.get(prediction.getEnd() - 1).getEnd(); - AnnotationFS annotation = aCas.createAnnotation(predictedType, begin, end); + var annotation = aCas.createAnnotation(predictedType, begin, end); annotation.setStringValue(predictedFeature, label); if (scoreFeature != null) { annotation.setDoubleValue(scoreFeature, prediction.getProb()); @@ -167,7 +172,7 @@ public Range predict(RecommenderContext aContext, CAS aCas, int aBegin, int aEnd } } - return new Range(units); + return rangeCoveringAnnotations(units); } @Override @@ -180,11 +185,11 @@ public int estimateSampleCount(List aCasses) public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) throws RecommendationException { - List data = extractNameSamples(aCasses); - List trainingSet = new ArrayList<>(); - List testSet = new ArrayList<>(); + var data = extractNameSamples(aCasses); + var trainingSet = new ArrayList(); + var testSet = new ArrayList(); - for (NameSample nameSample : data) { + for (var nameSample : data) { switch (aDataSplitter.getTargetSet(nameSample)) { case TRAIN: trainingSet.add(nameSample); @@ -198,28 +203,22 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) } } - int testSetSize = testSet.size(); - int trainingSetSize = trainingSet.size(); - double overallTrainingSize = data.size() - testSetSize; - double trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0; - - final int minTrainingSetSize = 2; - final int minTestSetSize = 2; - if (trainingSetSize < minTrainingSetSize || testSetSize < minTestSetSize) { - if ((getRecommender().getThreshold() <= 0.0d)) { - return new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), - SAMPLE_UNIT.getSimpleName()); - } + var testSetSize = testSet.size(); + var trainingSetSize = trainingSet.size(); + var overallTrainingSize = data.size() - testSetSize; + var trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0; - String info = String.format( - "Not enough evaluation data: training set [%s] sentences, test set [%s] of total [%s]", - trainingSetSize, testSetSize, data.size()); - LOG.info(info); + if (trainingSetSize < MIN_TRAINING_SET_SIZE || testSetSize < MIN_TEST_SET_SIZE) { + String msg = String.format( + "Not enough evaluation data: training set size [%d] (min. %d), test set size [%d] (min. %d) of total [%d] (min. %d)", + trainingSetSize, MIN_TRAINING_SET_SIZE, testSetSize, MIN_TEST_SET_SIZE, + data.size(), (MIN_TRAINING_SET_SIZE + MIN_TEST_SET_SIZE)); + LOG.info(msg); - EvaluationResult result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), + var result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), SAMPLE_UNIT.getSimpleName(), trainingSetSize, testSetSize, trainRatio); result.setEvaluationSkipped(true); - result.setErrorMsg(info); + result.setErrorMsg(msg); return result; } @@ -227,21 +226,21 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) testSet.size(), data.size()); // Train model - TokenNameFinderModel model = train(trainingSet, traits.getParameters()); - NameFinderME nameFinder = new NameFinderME(model); + var model = train(trainingSet, traits.getParameters()); + var nameFinder = new NameFinderME(model); // Evaluate - List labelPairs = new ArrayList<>(); - for (NameSample sample : testSet) { + var labelPairs = new ArrayList(); + for (var sample : testSet) { // clear adaptive data from feature generators if necessary if (sample.isClearAdaptiveDataSet()) { nameFinder.clearAdaptiveData(); } // Span contains one NE, Array of them all in one sentence - String[] sentence = sample.getSentence(); - Span[] predictedNames = nameFinder.find(sentence); - Span[] goldNames = sample.getNames(); + var sentence = sample.getSentence(); + var predictedNames = nameFinder.find(sentence); + var goldNames = sample.getNames(); labelPairs.addAll(determineLabelsForASentence(sentence, predictedNames, goldNames)); } @@ -309,7 +308,7 @@ private String determineLabel(Span aName, int aTokenIdx) private List extractNameSamples(List aCasses) { - List nameSamples = new ArrayList<>(); + var nameSamples = new ArrayList(); casses: for (CAS cas : aCasses) { Type sampleUnitType = getType(cas, SAMPLE_UNIT); diff --git a/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/pos/OpenNlpPosRecommender.java b/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/pos/OpenNlpPosRecommender.java index b4601283b9f..ef7dd0d7230 100644 --- a/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/pos/OpenNlpPosRecommender.java +++ b/inception/inception-imls-opennlp/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/opennlp/pos/OpenNlpPosRecommender.java @@ -59,7 +59,6 @@ import opennlp.tools.postag.POSSample; import opennlp.tools.postag.POSTaggerFactory; import opennlp.tools.postag.POSTaggerME; -import opennlp.tools.util.Sequence; import opennlp.tools.util.TrainingParameters; public class OpenNlpPosRecommender @@ -73,6 +72,9 @@ public class OpenNlpPosRecommender private static final Class SAMPLE_UNIT = Sentence.class; private static final Class DATAPOINT_UNIT = Token.class; + private static final int MIN_TRAINING_SET_SIZE = 2; + private static final int MIN_TEST_SET_SIZE = 2; + private final OpenNlpPosRecommenderTraits traits; public OpenNlpPosRecommender(Recommender aRecommender, OpenNlpPosRecommenderTraits aTraits) @@ -91,7 +93,7 @@ public boolean isReadyForPrediction(RecommenderContext aContext) @Override public void train(RecommenderContext aContext, List aCasses) throws RecommendationException { - List posSamples = extractPosSamples(aCasses); + var posSamples = extractPosSamples(aCasses); if (posSamples.size() < 2) { aContext.warn("Not enough training data: [%d] items", posSamples.size()); @@ -101,9 +103,9 @@ public void train(RecommenderContext aContext, List aCasses) throws Recomme // The beam size controls how many results are returned at most. But even if the user // requests only few results, we always use at least the default bean size recommended by // OpenNLP - int beamSize = Math.max(maxRecommendations, POSTaggerME.DEFAULT_BEAM_SIZE); + var beamSize = Math.max(maxRecommendations, POSTaggerME.DEFAULT_BEAM_SIZE); - TrainingParameters params = traits.getParameters(); + var params = traits.getParameters(); params.put(BeamSearch.BEAM_SIZE_PARAMETER, Integer.toString(beamSize)); POSModel model = train(posSamples, params); @@ -120,60 +122,60 @@ public TrainingCapability getTrainingCapability() public Range predict(RecommenderContext aContext, CAS aCas, int aBegin, int aEnd) throws RecommendationException { - POSModel model = aContext.get(KEY_MODEL).orElseThrow( + var model = aContext.get(KEY_MODEL).orElseThrow( () -> new RecommendationException("Key [" + KEY_MODEL + "] not found in context")); - POSTaggerME tagger = new POSTaggerME(model); + var tagger = new POSTaggerME(model); - Type sampleUnitType = getType(aCas, SAMPLE_UNIT); - Type predictedType = getPredictedType(aCas); - Type tokenType = getType(aCas, Token.class); + var sampleUnitType = getType(aCas, SAMPLE_UNIT); + var predictedType = getPredictedType(aCas); + var tokenType = getType(aCas, Token.class); - Feature scoreFeature = getScoreFeature(aCas); - Feature predictedFeature = getPredictedFeature(aCas); - Feature isPredictionFeature = getIsPredictionFeature(aCas); + var scoreFeature = getScoreFeature(aCas); + var predictedFeature = getPredictedFeature(aCas); + var isPredictionFeature = getIsPredictionFeature(aCas); var units = selectOverlapping(aCas, sampleUnitType, aBegin, aEnd); int predictionCount = 0; - for (AnnotationFS sampleUnit : units) { + for (var unit : units) { if (predictionCount >= traits.getPredictionLimit()) { break; } predictionCount++; - List tokenAnnotations = selectCovered(tokenType, sampleUnit); - String[] tokens = tokenAnnotations.stream() // + var tokenAnnotations = selectCovered(tokenType, unit); + var tokens = tokenAnnotations.stream() // .map(AnnotationFS::getCoveredText) // .toArray(String[]::new); - Sequence[] bestSequences = tagger.topKSequences(tokens); + var bestSequences = tagger.topKSequences(tokens); // LOG.debug("Total number of sequences predicted: {}", bestSequences.length); for (int s = 0; s < Math.min(bestSequences.length, maxRecommendations); s++) { - Sequence sequence = bestSequences[s]; - List outcomes = sequence.getOutcomes(); - double[] probabilities = sequence.getProbs(); + var sequence = bestSequences[s]; + var outcomes = sequence.getOutcomes(); + var probabilities = sequence.getProbs(); // LOG.debug("Sequence {} score {}", s, sequence.getScore()); // LOG.debug("Outcomes: {}", outcomes); // LOG.debug("Probabilities: {}", asList(probabilities)); for (int i = 0; i < outcomes.size(); i++) { - String label = outcomes.get(i); + var label = outcomes.get(i); // Do not return PADded tokens if (PAD.equals(label)) { continue; } - AnnotationFS token = tokenAnnotations.get(i); + var token = tokenAnnotations.get(i); int begin = token.getBegin(); int end = token.getEnd(); double score = probabilities[i]; // Create the prediction - AnnotationFS annotation = aCas.createAnnotation(predictedType, begin, end); + var annotation = aCas.createAnnotation(predictedType, begin, end); annotation.setStringValue(predictedFeature, label); annotation.setDoubleValue(scoreFeature, score); annotation.setBooleanValue(isPredictionFeature, true); @@ -182,7 +184,7 @@ public Range predict(RecommenderContext aContext, CAS aCas, int aBegin, int aEnd } } - return new Range(units); + return Range.rangeCoveringAnnotations(units); } @Override @@ -195,11 +197,11 @@ public int estimateSampleCount(List aCasses) public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) throws RecommendationException { - List data = extractPosSamples(aCasses); - List trainingSet = new ArrayList<>(); - List testSet = new ArrayList<>(); + var data = extractPosSamples(aCasses); + var trainingSet = new ArrayList(); + var testSet = new ArrayList(); - for (POSSample posSample : data) { + for (var posSample : data) { switch (aDataSplitter.getTargetSet(posSample)) { case TRAIN: trainingSet.add(posSample); @@ -213,29 +215,22 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) } } - int testSetSize = testSet.size(); - int trainingSetSize = trainingSet.size(); - double overallTrainingSize = data.size() - testSetSize; - double trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0; - - final int minTrainingSetSize = 2; - final int minTestSetSize = 2; - if (trainingSetSize < minTrainingSetSize || testSetSize < minTestSetSize) { - if ((getRecommender().getThreshold() <= 0.0d)) { - return new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), - SAMPLE_UNIT.getSimpleName()); - } + var testSetSize = testSet.size(); + var trainingSetSize = trainingSet.size(); + var overallTrainingSize = data.size() - testSetSize; + var trainRatio = (overallTrainingSize > 0) ? trainingSetSize / overallTrainingSize : 0.0; - String info = String.format( + if (trainingSetSize < MIN_TRAINING_SET_SIZE || testSetSize < MIN_TEST_SET_SIZE) { + var msg = String.format( "Not enough evaluation data: training set size [%d] (min. %d), test set size [%d] (min. %d) of total [%d] (min. %d)", - trainingSetSize, minTrainingSetSize, testSetSize, minTestSetSize, data.size(), - (minTrainingSetSize + minTestSetSize)); - LOG.info(info); + trainingSetSize, MIN_TRAINING_SET_SIZE, testSetSize, MIN_TEST_SET_SIZE, + data.size(), (MIN_TRAINING_SET_SIZE + MIN_TEST_SET_SIZE)); + LOG.info(msg); - EvaluationResult result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), + var result = new EvaluationResult(DATAPOINT_UNIT.getSimpleName(), SAMPLE_UNIT.getSimpleName(), trainingSetSize, testSetSize, trainRatio); result.setEvaluationSkipped(true); - result.setErrorMsg(info); + result.setErrorMsg(msg); return result; } @@ -243,7 +238,7 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) testSet.size(), data.size()); // Train model - POSModel model = train(trainingSet, traits.getParameters()); + var model = train(trainingSet, traits.getParameters()); if (model == null) { throw new RecommendationException("Model is null, cannot evaluate!"); } @@ -251,8 +246,8 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) POSTaggerME tagger = new POSTaggerME(model); // Evaluate - List labelPairs = new ArrayList<>(); - for (POSSample sample : testSet) { + var labelPairs = new ArrayList(); + for (var sample : testSet) { String[] predictedTags = tagger.tag(sample.getSentence()); String[] goldTags = sample.getTags(); for (int i = 0; i < predictedTags.length; i++) { @@ -266,13 +261,13 @@ public EvaluationResult evaluate(List aCasses, DataSplitter aDataSplitter) private List extractPosSamples(List aCasses) { - List posSamples = new ArrayList<>(); + var posSamples = new ArrayList(); casses: for (CAS cas : aCasses) { - Type sampleUnitType = getType(cas, SAMPLE_UNIT); - Type tokenType = getType(cas, Token.class); + var sampleUnitType = getType(cas, SAMPLE_UNIT); + var tokenType = getType(cas, Token.class); - for (Annotation sampleUnit : cas. select(sampleUnitType)) { + for (var sampleUnit : cas. select(sampleUnitType)) { if (posSamples.size() >= traits.getTrainingSetSizeLimit()) { break casses; } @@ -281,8 +276,7 @@ private List extractPosSamples(List aCasses) continue; } - List tokens = cas. select(tokenType).coveredBy(sampleUnit) - .asList(); + var tokens = cas. select(tokenType).coveredBy(sampleUnit).asList(); createPosSample(cas, sampleUnit, tokens).map(posSamples::add); } @@ -296,19 +290,19 @@ private List extractPosSamples(List aCasses) private Optional createPosSample(CAS aCas, AnnotationFS aSentence, Collection aTokens) { - Type annotationType = getType(aCas, layerName); - Feature feature = annotationType.getFeatureByBaseName(featureName); + var annotationType = getType(aCas, layerName); + var feature = annotationType.getFeatureByBaseName(featureName); - int numberOfTokens = aTokens.size(); - String[] tokens = new String[numberOfTokens]; - String[] tags = new String[numberOfTokens]; + var numberOfTokens = aTokens.size(); + var tokens = new String[numberOfTokens]; + var tags = new String[numberOfTokens]; - int withTagCount = 0; + var withTagCount = 0; - int i = 0; - for (AnnotationFS token : aTokens) { + var i = 0; + for (var token : aTokens) { tokens[i] = token.getCoveredText(); - String tag = getFeatureValueCovering(aCas, token, annotationType, feature); + var tag = getFeatureValueCovering(aCas, token, annotationType, feature); tags[i] = tag; // If the tag is neither PAD nor null, then there is at @@ -322,7 +316,7 @@ private Optional createPosSample(CAS aCas, AnnotationFS aSentence, // Require at least X percent of the sentence to have tags to avoid class imbalance on PAD // tag. - double coverage = ((double) withTagCount * 100) / (double) numberOfTokens; + var coverage = ((double) withTagCount * 100) / (double) numberOfTokens; if (coverage >= traits.getTaggedTokensThreshold()) { return Optional.of(new POSSample(tokens, tags)); } @@ -334,13 +328,13 @@ private Optional createPosSample(CAS aCas, AnnotationFS aSentence, private String getFeatureValueCovering(CAS aCas, AnnotationFS aToken, Type aType, Feature aFeature) { - List annotations = CasUtil.selectCovered(aType, aToken); + var annotations = CasUtil.selectCovered(aType, aToken); if (annotations.isEmpty()) { return PAD; } - String value = annotations.get(0).getFeatureValueAsString(aFeature); + var value = annotations.get(0).getFeatureValueAsString(aFeature); return isNoneBlank(value) ? value : PAD; } @@ -352,8 +346,8 @@ private POSModel train(List aPosSamples, TrainingParameters aParamete return null; } - try (POSSampleStream stream = new POSSampleStream(aPosSamples)) { - POSTaggerFactory taggerFactory = new POSTaggerFactory(); + try (var stream = new POSSampleStream(aPosSamples)) { + var taggerFactory = new POSTaggerFactory(); return POSTaggerME.train("unknown", stream, aParameters, taggerFactory); } catch (IOException e) { diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/AnnotationSuggestion.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/AnnotationSuggestion.java index 0e8d779bd3a..f2dad5f6335 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/AnnotationSuggestion.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/AnnotationSuggestion.java @@ -33,6 +33,8 @@ public abstract class AnnotationSuggestion { private static final long serialVersionUID = -7137765759688480950L; + public static final int NEW_ID = -1; + public static final String EXTENSION_ID = "rec"; /** @@ -71,6 +73,7 @@ public abstract class AnnotationSuggestion public static final int FLAG_ALL = FLAG_OVERLAP | FLAG_SKIPPED | FLAG_REJECTED | FLAG_TRANSIENT_ACCEPTED | FLAG_TRANSIENT_REJECTED | FLAG_TRANSIENT_CORRECTED; + protected final int generation; protected final int id; protected final long recommenderId; protected final String recommenderName; @@ -84,11 +87,15 @@ public abstract class AnnotationSuggestion private AutoAcceptMode autoAcceptMode; private int hidingFlags = 0; + private int age = 0; - public AnnotationSuggestion(int aId, long aRecommenderId, String aRecommenderName, - long aLayerId, String aFeature, String aDocumentName, String aLabel, String aUiLabel, - double aScore, String aScoreExplanation, AutoAcceptMode aAutoAcceptMode) + public AnnotationSuggestion(int aId, int aGeneration, int aAge, long aRecommenderId, + String aRecommenderName, long aLayerId, String aFeature, String aDocumentName, + String aLabel, String aUiLabel, double aScore, String aScoreExplanation, + AutoAcceptMode aAutoAcceptMode, int aHidingFlags) { + generation = aGeneration; + age = aAge; label = aLabel; uiLabel = aUiLabel; id = aId; @@ -100,21 +107,7 @@ public AnnotationSuggestion(int aId, long aRecommenderId, String aRecommenderNam recommenderId = aRecommenderId; documentName = aDocumentName; autoAcceptMode = aAutoAcceptMode; - } - - public AnnotationSuggestion(AnnotationSuggestion aObject) - { - label = aObject.label; - uiLabel = aObject.uiLabel; - id = aObject.id; - layerId = aObject.layerId; - feature = aObject.feature; - recommenderName = aObject.recommenderName; - score = aObject.score; - scoreExplanation = aObject.scoreExplanation; - recommenderId = aObject.recommenderId; - documentName = aObject.documentName; - autoAcceptMode = aObject.autoAcceptMode; + hidingFlags = aHidingFlags; } public int getId() @@ -184,6 +177,11 @@ public void show(int aFlags) hidingFlags &= ~aFlags; } + protected int getHidingFlags() + { + return hidingFlags; + } + public String getReasonForHiding() { StringBuilder sb = new StringBuilder(); @@ -297,4 +295,24 @@ public boolean hideSuggestion(LearningRecordType aAction) return false; } } + + public int incrementAge() + { + age++; + return age; + } + + public int getAge() + { + return age; + } + + /** + * @return a clone of the current suggestion with the new ID. This is used when adding a + * suggestion to {@link Predictions} if the ID of the suggestion is set to + * {@link #NEW_ID}. + * @param aId + * the ID of the suggestion. + */ + abstract public AnnotationSuggestion assignId(int aId); } diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/ExtendedId.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/ExtendedId.java index ff46d1c02eb..361305ee39d 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/ExtendedId.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/ExtendedId.java @@ -25,22 +25,21 @@ class ExtendedId { private static final long serialVersionUID = -5214683455382881005L; - private final String documentName; + private final int suggestionId; + private final long recommenderId; private final long layerId; + private final String documentName; private final Position position; - private final int annotationId; - private final long recommenderId; private final int hash; - public ExtendedId(String documentName, long layerId, Position aPosition, long recommenderId, - int annotationId) + public ExtendedId(AnnotationSuggestion aSuggestion) { - this.documentName = documentName; - this.layerId = layerId; - this.annotationId = annotationId; - this.recommenderId = recommenderId; - this.position = aPosition; - hash = Objects.hash(annotationId, documentName, layerId, position, recommenderId); + documentName = aSuggestion.getDocumentName(); + layerId = aSuggestion.getLayerId(); + suggestionId = aSuggestion.getId(); + recommenderId = aSuggestion.getRecommenderId(); + position = aSuggestion.getPosition(); + hash = Objects.hash(suggestionId, documentName, layerId, position, recommenderId); } public String getDocumentName() @@ -58,9 +57,9 @@ public Position getPosition() return position; } - public int getAnnotationId() + public int getSuggestionId() { - return annotationId; + return suggestionId; } public long getRecommenderId() @@ -91,7 +90,7 @@ public boolean equals(Object obj) ExtendedId other = (ExtendedId) obj; return Objects.equals(position, other.position) // - && annotationId == other.annotationId // + && suggestionId == other.suggestionId // && Objects.equals(documentName, other.documentName) // && layerId == other.layerId // && recommenderId == other.recommenderId; diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Predictions.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Predictions.java index b0e25f40cf5..d62fa1ff715 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Predictions.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Predictions.java @@ -53,25 +53,41 @@ public class Predictions { private static final long serialVersionUID = -1598768729246662885L; + private final int generation; private final Project project; private final User sessionOwner; private final String dataOwner; private final Map> idxDocuments = new HashMap<>(); - private final Object predictions = new Object(); + private final Object predictionsLock = new Object(); private final Set seenDocumentsForPrediction = new HashSet<>(); private final List log = new ArrayList<>(); + // Predictions are (currently) scoped to a user session. We assume that within a single user + // session, the pool of IDs of positive integer values is never exhausted. + private int nextId; + public Predictions(User aSessionOwner, String aDataOwner, Project aProject) { Validate.notNull(aProject, "Project must be specified"); Validate.notNull(aSessionOwner, "Session owner must be specified"); - Validate.notNull(aSessionOwner, "Data owner must be specified"); + Validate.notNull(aDataOwner, "Data owner must be specified"); project = aProject; sessionOwner = aSessionOwner; dataOwner = aDataOwner; + nextId = 0; + generation = 1; + } + + public Predictions(Predictions aPredecessor) + { + project = aPredecessor.project; + sessionOwner = aPredecessor.sessionOwner; + dataOwner = aPredecessor.dataOwner; + nextId = aPredecessor.nextId; + generation = aPredecessor.generation + 1; } public User getSessionOwner() @@ -151,7 +167,7 @@ public SuggestionDocumentGroup getGroupedPre private List getFlattenedPredictions(Class type, String aDocumentName, AnnotationLayer aLayer, int aWindowBegin, int aWindowEnd) { - synchronized (predictions) { + synchronized (predictionsLock) { var byDocument = idxDocuments.getOrDefault(aDocumentName, emptyMap()); return byDocument.entrySet().stream() // .filter(f -> type.isInstance(f.getValue())) // @@ -180,11 +196,11 @@ private List getFlattenedPredictions(Class getPredictionByVID(SourceDocument aDocument, VID aVID) { - synchronized (predictions) { + synchronized (predictionsLock) { var byDocument = idxDocuments.getOrDefault(aDocument.getName(), emptyMap()); return byDocument.values().stream() // - .filter(f -> f.getId() == aVID.getSubId()) // - .filter(f -> f.getRecommenderId() == aVID.getId()) // + .filter(suggestion -> suggestion.getId() == aVID.getSubId()) // + .filter(suggestion -> suggestion.getRecommenderId() == aVID.getId()) // .findFirst(); } } @@ -195,11 +211,19 @@ public Optional getPredictionByVID(SourceDocument aDocumen */ public void putPredictions(List aPredictions) { - synchronized (predictions) { + synchronized (predictionsLock) { for (var prediction : aPredictions) { - var xid = new ExtendedId(prediction.getDocumentName(), prediction.getLayerId(), - prediction.getPosition(), prediction.getRecommenderId(), - prediction.getId()); + // Assign ID to predictions that do not have an ID yet + if (prediction.getId() == AnnotationSuggestion.NEW_ID) { + prediction = prediction.assignId(nextId); + nextId++; + if (nextId < 0) { + throw new IllegalStateException( + "Annotation suggestion ID overflow. Restart session."); + } + } + + var xid = new ExtendedId(prediction); var byDocument = idxDocuments.computeIfAbsent(prediction.getDocumentName(), $ -> new HashMap<>()); byDocument.put(xid, prediction); @@ -212,35 +236,23 @@ public Project getProject() return project; } - /** - * @return whether there are any predictions. - * @deprecated Use {@link #isEmpty()} instead. - */ - @Deprecated - public boolean hasPredictions() - { - synchronized (predictions) { - return !idxDocuments.isEmpty(); - } - } - public boolean isEmpty() { - synchronized (predictions) { + synchronized (predictionsLock) { return idxDocuments.values().stream().allMatch(Map::isEmpty); } } public int size() { - synchronized (predictions) { + synchronized (predictionsLock) { return idxDocuments.values().stream().mapToInt(Map::size).sum(); } } public void removePredictions(Long recommenderId) { - synchronized (predictions) { + synchronized (predictionsLock) { idxDocuments.values().forEach(docGroup -> docGroup.entrySet() // .removeIf((p) -> p.getKey().getRecommenderId() == recommenderId)); } @@ -249,7 +261,7 @@ public void removePredictions(Long recommenderId) @SuppressWarnings({ "rawtypes", "unchecked" }) public List getAlternativeSuggestions(SpanSuggestion aSuggestion) { - synchronized (predictions) { + synchronized (predictionsLock) { var byDocument = idxDocuments.getOrDefault(aSuggestion.getDocumentName(), emptyMap()); return byDocument.entrySet().stream() // .filter(f -> f.getValue() instanceof SpanSuggestion) // @@ -284,7 +296,7 @@ public List getAlternativeSuggestions(SpanSuggestion aSuggestion public List getPredictionsByTokenAndFeature(String aDocumentName, AnnotationLayer aLayer, int aBegin, int aEnd, String aFeature) { - synchronized (predictions) { + synchronized (predictionsLock) { var byDocument = idxDocuments.getOrDefault(aDocumentName, emptyMap()); return byDocument.entrySet().stream() // .filter(f -> f.getValue() instanceof SpanSuggestion) // @@ -301,7 +313,7 @@ public List getPredictionsByTokenAndFeature(String aDocumentName public List getPredictionsByRecommenderAndDocument( Recommender aRecommender, String aDocumentName) { - synchronized (predictions) { + synchronized (predictionsLock) { var byDocument = idxDocuments.getOrDefault(aDocumentName, emptyMap()); return byDocument.entrySet().stream() // .filter(f -> f.getKey().getRecommenderId() == (long) aRecommender.getId()) @@ -312,7 +324,7 @@ public List getPredictionsByRecommenderAndDocument( public List getPredictionsByDocument(String aDocumentName) { - synchronized (predictions) { + synchronized (predictionsLock) { var byDocument = idxDocuments.getOrDefault(aDocumentName, emptyMap()); return byDocument.entrySet().stream() // .map(Map.Entry::getValue) // @@ -353,6 +365,11 @@ public void inheritLog(List aLogMessages) } } + public int getGeneration() + { + return generation; + } + public List getLog() { synchronized (log) { diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Recommender.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Recommender.java index 7e458f0fef1..6d1dbf94e41 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Recommender.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/Recommender.java @@ -195,6 +195,11 @@ public double getThreshold() return threshold; } + /** + * Activation score threshold. + * + * @param aThreshold + */ public void setThreshold(double aThreshold) { threshold = aThreshold; diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/RelationSuggestion.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/RelationSuggestion.java index f17133e6e5c..ca1a7d22730 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/RelationSuggestion.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/RelationSuggestion.java @@ -20,7 +20,6 @@ import java.io.Serializable; import org.apache.commons.lang3.builder.ToStringBuilder; -import org.apache.uima.cas.text.AnnotationFS; public class RelationSuggestion extends AnnotationSuggestion @@ -32,47 +31,29 @@ public class RelationSuggestion private RelationSuggestion(Builder builder) { - super(builder.id, builder.recommenderId, builder.recommenderName, builder.layerId, - builder.feature, builder.documentName, builder.label, builder.uiLabel, - builder.score, builder.scoreExplanation, builder.autoAcceptMode); + super(builder.id, builder.generation, builder.age, builder.recommenderId, + builder.recommenderName, builder.layerId, builder.feature, builder.documentName, + builder.label, builder.uiLabel, builder.score, builder.scoreExplanation, + builder.autoAcceptMode, builder.hidingFlags); this.position = builder.position; } - public RelationSuggestion(int aId, Recommender aRecommender, long aLayerId, String aFeature, - String aDocumentName, AnnotationFS aSource, AnnotationFS aTarget, String aLabel, - String aUiLabel, double aScore, String aScoreExplanation, - AutoAcceptMode aAutoAcceptMode) - { - this(aId, aRecommender.getId(), aRecommender.getName(), aLayerId, aFeature, aDocumentName, - aSource.getBegin(), aSource.getEnd(), aTarget.getBegin(), aTarget.getEnd(), aLabel, - aUiLabel, aScore, aScoreExplanation, aAutoAcceptMode); - } - + /** + * @deprecated Use builder instead + */ + @Deprecated public RelationSuggestion(int aId, long aRecommenderId, String aRecommenderName, long aLayerId, String aFeature, String aDocumentName, int aSourceBegin, int aSourceEnd, int aTargetBegin, int aTargetEnd, String aLabel, String aUiLabel, double aScore, String aScoreExplanation, AutoAcceptMode aAutoAcceptMode) { - super(aId, aRecommenderId, aRecommenderName, aLayerId, aFeature, aDocumentName, aLabel, - aUiLabel, aScore, aScoreExplanation, aAutoAcceptMode); + super(aId, 0, 0, aRecommenderId, aRecommenderName, aLayerId, aFeature, aDocumentName, + aLabel, aUiLabel, aScore, aScoreExplanation, aAutoAcceptMode, 0); position = new RelationPosition(aSourceBegin, aSourceEnd, aTargetBegin, aTargetEnd); } - /** - * Copy constructor. - * - * @param aObject - * The annotationObject to copy - */ - public RelationSuggestion(RelationSuggestion aObject) - { - super(aObject); - - position = new RelationPosition(aObject.position); - } - // Getter and setter @Override @@ -100,14 +81,31 @@ public int getWindowEnd() @Override public String toString() { - return new ToStringBuilder(this).append("id", id).append("recommenderId", recommenderId) - .append("recommenderName", recommenderName).append("layerId", layerId) - .append("feature", feature).append("documentName", documentName) + return new ToStringBuilder(this) // + .append("id", id) // + .append("generation", generation) // + .append("age", getAge()) // + .append("recommenderId", recommenderId) // + .append("recommenderName", recommenderName) // + .append("layerId", layerId) // + .append("feature", feature) // + .append("documentName", documentName) // .append("position", position) // - .append("windowBegin", getWindowBegin()).append("windowEnd", getWindowEnd()) // - .append("label", label).append("uiLabel", uiLabel).append("score", score) - .append("confindenceExplanation", scoreExplanation).append("visible", isVisible()) - .append("reasonForHiding", getReasonForHiding()).toString(); + .append("windowBegin", getWindowBegin()) // + .append("windowEnd", getWindowEnd()) // + .append("label", label) // + .append("uiLabel", uiLabel) // + .append("score", score) // + .append("confindenceExplanation", scoreExplanation) // + .append("visible", isVisible()) // + .append("reasonForHiding", getReasonForHiding()) // + .toString(); + } + + @Override + public AnnotationSuggestion assignId(int aId) + { + return toBuilder().withId(aId).build(); } public static Builder builder() @@ -115,8 +113,30 @@ public static Builder builder() return new Builder(); } + public Builder toBuilder() + { + return builder() // + .withId(id) // + .withGeneration(generation) // + .withAge(getAge()) // + .withRecommenderId(recommenderId) // + .withRecommenderName(recommenderName) // + .withLayerId(layerId) // + .withFeature(feature) // + .withDocumentName(documentName) // + .withLabel(label) // + .withUiLabel(uiLabel) // + .withScore(score) // + .withScoreExplanation(scoreExplanation) // + .withPosition(position) // + .withAutoAcceptMode(getAutoAcceptMode()) // + .withHidingFlags(getHidingFlags()); + } + public static final class Builder { + private int generation; + private int age; private int id; private long recommenderId; private String recommenderName; @@ -129,6 +149,7 @@ public static final class Builder private String scoreExplanation; private RelationPosition position; private AutoAcceptMode autoAcceptMode; + private int hidingFlags; private Builder() { @@ -140,6 +161,18 @@ public Builder withId(int aId) return this; } + public Builder withGeneration(int aGeneration) + { + this.generation = aGeneration; + return this; + } + + public Builder withAge(int aAge) + { + this.age = aAge; + return this; + } + public Builder withRecommender(Recommender aRecommender) { this.recommenderId = aRecommender.getId(); @@ -215,6 +248,12 @@ public Builder withAutoAcceptMode(AutoAcceptMode aAutoAcceptMode) return this; } + public Builder withHidingFlags(int aFlags) + { + this.hidingFlags = aFlags; + return this; + } + public RelationSuggestion build() { return new RelationSuggestion(this); diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/SpanSuggestion.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/SpanSuggestion.java index 3524819a1b8..d6041865c59 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/SpanSuggestion.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/SpanSuggestion.java @@ -32,50 +32,31 @@ public class SpanSuggestion private SpanSuggestion(Builder builder) { - super(builder.id, builder.recommenderId, builder.recommenderName, builder.layerId, - builder.feature, builder.documentName, builder.label, builder.uiLabel, - builder.score, builder.scoreExplanation, builder.autoAcceptMode); + super(builder.id, builder.generation, builder.age, builder.recommenderId, + builder.recommenderName, builder.layerId, builder.feature, builder.documentName, + builder.label, builder.uiLabel, builder.score, builder.scoreExplanation, + builder.autoAcceptMode, builder.hidingFlags); position = builder.position; coveredText = builder.coveredText; } - public SpanSuggestion(int aId, Recommender aRecommender, long aLayerId, String aFeature, - String aDocumentName, Offset aOffset, String aCoveredText, String aLabel, - String aUiLabel, double aScore, String aScoreExplanation, - AutoAcceptMode aAutoAcceptMode) - { - this(aId, aRecommender.getId(), aRecommender.getName(), aLayerId, aFeature, aDocumentName, - aOffset.getBegin(), aOffset.getEnd(), aCoveredText, aLabel, aUiLabel, aScore, - aScoreExplanation, aAutoAcceptMode); - } - + /** + * @deprecated Use builder instead. + */ + @Deprecated public SpanSuggestion(int aId, long aRecommenderId, String aRecommenderName, long aLayerId, String aFeature, String aDocumentName, int aBegin, int aEnd, String aCoveredText, String aLabel, String aUiLabel, double aScore, String aScoreExplanation, AutoAcceptMode aAutoAcceptMode) { - super(aId, aRecommenderId, aRecommenderName, aLayerId, aFeature, aDocumentName, aLabel, - aUiLabel, aScore, aScoreExplanation, aAutoAcceptMode); + super(aId, 0, 0, aRecommenderId, aRecommenderName, aLayerId, aFeature, aDocumentName, + aLabel, aUiLabel, aScore, aScoreExplanation, aAutoAcceptMode, 0); position = new Offset(aBegin, aEnd); coveredText = aCoveredText; } - /** - * Copy constructor. - * - * @param aObject - * The annotationObject to copy - */ - public SpanSuggestion(SpanSuggestion aObject) - { - super(aObject); - - position = new Offset(aObject.position.getBegin(), aObject.position.getEnd()); - coveredText = aObject.coveredText; - } - // Getter and setter public String getCoveredText() @@ -114,20 +95,44 @@ public Offset getPosition() @Override public String toString() { - return new ToStringBuilder(this).append("id", id).append("recommenderId", recommenderId) - .append("recommenderName", recommenderName).append("layerId", layerId) - .append("feature", feature).append("documentName", documentName) - .append("position", position).append("coveredText", coveredText) - .append("label", label).append("uiLabel", uiLabel).append("score", score) - .append("confindenceExplanation", scoreExplanation).append("visible", isVisible()) - .append("reasonForHiding", getReasonForHiding()) - .append("autoAcceptMode", getAutoAcceptMode()).toString(); + return new ToStringBuilder(this) // + .append("id", id) // + .append("generation", generation) // + .append("age", getAge()) // + .append("recommenderId", recommenderId) // + .append("recommenderName", recommenderName) // + .append("layerId", layerId) // + .append("feature", feature) // + .append("documentName", documentName) // + .append("position", position) // + .append("coveredText", coveredText) // + .append("label", label) // + .append("uiLabel", uiLabel) // + .append("score", score) // + .append("confindenceExplanation", scoreExplanation) // + .append("visible", isVisible()) // + .append("reasonForHiding", getReasonForHiding()) // + .append("autoAcceptMode", getAutoAcceptMode()) // + .toString(); + } + + @Override + public AnnotationSuggestion assignId(int aId) + { + return toBuilder().withId(aId).build(); + } + + public static Builder builder() + { + return new Builder(); } public Builder toBuilder() { return builder() // .withId(id) // + .withGeneration(generation) // + .withAge(getAge()) // .withRecommenderId(recommenderId) // .withRecommenderName(recommenderName) // .withLayerId(layerId) // @@ -139,17 +144,15 @@ public Builder toBuilder() .withScoreExplanation(scoreExplanation) // .withPosition(position) // .withCoveredText(coveredText) // - .withAutoAcceptMode(getAutoAcceptMode()); - } - - public static Builder builder() - { - return new Builder(); + .withAutoAcceptMode(getAutoAcceptMode()) // + .withHidingFlags(getHidingFlags()); } public static final class Builder { private int id; + private int generation; + private int age; private long recommenderId; private String recommenderName; private long layerId; @@ -162,6 +165,7 @@ public static final class Builder private Offset position; private String coveredText; private AutoAcceptMode autoAcceptMode; + private int hidingFlags; private Builder() { @@ -173,6 +177,18 @@ public Builder withId(int aId) return this; } + public Builder withAge(int aAge) + { + this.age = aAge; + return this; + } + + public Builder withGeneration(int aGeneration) + { + this.generation = aGeneration; + return this; + } + public Builder withRecommender(Recommender aRecommender) { this.recommenderId = aRecommender.getId(); @@ -254,6 +270,12 @@ public Builder withAutoAcceptMode(AutoAcceptMode aAutoAcceptMode) return this; } + public Builder withHidingFlags(int aFlags) + { + this.hidingFlags = aFlags; + return this; + } + public SpanSuggestion build() { return new SpanSuggestion(this); diff --git a/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/PredictionsTest.java b/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/PredictionsTest.java index e1af2069d2e..d8659f3b67f 100644 --- a/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/PredictionsTest.java +++ b/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/PredictionsTest.java @@ -109,6 +109,35 @@ void timeGetGroupedPredictions() throws Exception } } + @Test + void thatIdsAreAssigned() throws Exception + { + var doc = "doc"; + sut = new Predictions(user, user.getUsername(), project); + sut.putPredictions(asList( // + SpanSuggestion.builder() // + .withId(AnnotationSuggestion.NEW_ID) // + .withDocumentName(doc) // + .build())); + + assertThat(sut.getPredictionsByDocument(doc)) // + .extracting(AnnotationSuggestion::getId) // + .containsExactly(0); + + var inheritedPredictions = sut.getPredictionsByDocument(doc); + sut = new Predictions(sut); + sut.putPredictions(asList( // + SpanSuggestion.builder() // + .withId(AnnotationSuggestion.NEW_ID) // + .withDocumentName(doc) // + .build())); + sut.putPredictions(inheritedPredictions); + + assertThat(sut.getPredictionsByDocument(doc)) // + .extracting(AnnotationSuggestion::getId) // + .containsExactlyInAnyOrder(0, 1); + } + private List generatePredictions(int aDocs, int aRecommenders, int aSuggestions) throws Exception diff --git a/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/RelationSuggestionTest.java b/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/RelationSuggestionTest.java new file mode 100644 index 00000000000..8dec84b08d6 --- /dev/null +++ b/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/RelationSuggestionTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.api.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class RelationSuggestionTest +{ + @Test + void thatCloningRetainsAllInformation() + { + var s = RelationSuggestion.builder() // + .withId(1) // + .withGeneration(2) // + .withAge(3) // + .withRecommenderId(4) // + .withRecommenderName("rec") // + .withLayerId(5) // + .withFeature("feature") // + .withDocumentName("document") // + .withLabel("label") // + .withUiLabel("uiLabel") // + .withScore(6.0) // + .withScoreExplanation("scoreExplanation") // + .withPosition(new RelationPosition(1, 2, 3, 4)) // + .withAutoAcceptMode(AutoAcceptMode.ON_FIRST_ACCESS) // + .withHidingFlags(7) // + .build(); + + assertThat(s.toBuilder().build()).usingRecursiveComparison().isEqualTo(s); + } +} diff --git a/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/SpanSuggestionTest.java b/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/SpanSuggestionTest.java new file mode 100644 index 00000000000..e21c1cb7ff5 --- /dev/null +++ b/inception/inception-recommendation-api/src/test/java/de/tudarmstadt/ukp/inception/recommendation/api/model/SpanSuggestionTest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.api.model; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class SpanSuggestionTest +{ + @Test + void thatCloningRetainsAllInformation() + { + var s = SpanSuggestion.builder() // + .withId(1) // + .withGeneration(2) // + .withAge(3) // + .withRecommenderId(4) // + .withRecommenderName("rec") // + .withLayerId(5) // + .withFeature("feature") // + .withDocumentName("document") // + .withLabel("label") // + .withUiLabel("uiLabel") // + .withScore(6.0) // + .withScoreExplanation("scoreExplanation") // + .withPosition(new Offset(1, 2)) // + .withCoveredText("coveredText") // + .withAutoAcceptMode(AutoAcceptMode.ON_FIRST_ACCESS) // + .withHidingFlags(7) // + .build(); + + assertThat(s.toBuilder().build()).usingRecursiveComparison().isEqualTo(s); + } +} diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java index fa7e72568d6..16028889b9e 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java @@ -40,9 +40,12 @@ import static de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionType.RELATION; import static de.tudarmstadt.ukp.inception.recommendation.api.model.SuggestionType.SPAN; import static de.tudarmstadt.ukp.inception.recommendation.api.recommender.TrainingCapability.TRAINING_NOT_SUPPORTED; +import static de.tudarmstadt.ukp.inception.rendering.model.Range.rangeCoveringDocument; import static java.lang.Math.max; import static java.lang.Math.min; +import static java.util.Collections.emptyList; import static java.util.Comparator.comparingInt; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableList; @@ -62,6 +65,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -131,6 +135,7 @@ import de.tudarmstadt.ukp.clarin.webanno.security.UserDao; import de.tudarmstadt.ukp.clarin.webanno.security.model.User; import de.tudarmstadt.ukp.clarin.webanno.support.StopWatch; +import de.tudarmstadt.ukp.clarin.webanno.support.WebAnnoConst; import de.tudarmstadt.ukp.clarin.webanno.support.logging.LogMessage; import de.tudarmstadt.ukp.clarin.webanno.support.logging.LogMessageGroup; import de.tudarmstadt.ukp.clarin.webanno.support.uima.ICasUtil; @@ -181,6 +186,7 @@ import de.tudarmstadt.ukp.inception.recommendation.tasks.SelectionTask; import de.tudarmstadt.ukp.inception.recommendation.tasks.TrainingTask; import de.tudarmstadt.ukp.inception.recommendation.util.OverlapIterator; +import de.tudarmstadt.ukp.inception.rendering.model.Range; import de.tudarmstadt.ukp.inception.scheduling.SchedulingService; import de.tudarmstadt.ukp.inception.scheduling.Task; import de.tudarmstadt.ukp.inception.scheduling.TaskMonitor; @@ -891,17 +897,21 @@ private void triggerRun(String aSessionOwner, Project aProject, String aEventNam @Override public List getLog(String aUser, Project aProject) { - Predictions activePredictions = getState(aUser, aProject).getActivePredictions(); - Predictions incomingPredictions = getState(aUser, aProject).getIncomingPredictions(); + var activePredictions = getState(aUser, aProject).getActivePredictions(); + var incomingPredictions = getState(aUser, aProject).getIncomingPredictions(); - List messageSets = new ArrayList<>(); + var messageSets = new ArrayList(); if (activePredictions != null) { - messageSets.add(new LogMessageGroup("Active", activePredictions.getLog())); + messageSets.add( + new LogMessageGroup("Active (gen. " + activePredictions.getGeneration() + ")", + activePredictions.getLog())); } if (incomingPredictions != null) { - messageSets.add(new LogMessageGroup("Incoming", incomingPredictions.getLog())); + messageSets.add(new LogMessageGroup( + "Incoming (gen. " + incomingPredictions.getGeneration() + ")", + incomingPredictions.getLog())); } return messageSets; @@ -1538,14 +1548,14 @@ public void removeLearningRecords(LearningRecord aRecord) } private void computePredictions(LazyCas aOriginalCas, - EvaluatedRecommender aEvaluatedRecommender, Predictions aPredictions, CAS predictionCas, - SourceDocument aDocument, User aSessionOwner, int aPredictionBegin, int aPredictionEnd) + EvaluatedRecommender aEvaluatedRecommender, Predictions activePredictions, + Predictions aPredictions, CAS predictionCas, SourceDocument aDocument, + User aSessionOwner, int aPredictionBegin, int aPredictionEnd) throws IOException { - Project project = aDocument.getProject(); - Predictions activePredictions = getPredictions(aSessionOwner, project); - int predictionBegin = aPredictionBegin; - int predictionEnd = aPredictionEnd; + var project = aDocument.getProject(); + var predictionBegin = aPredictionBegin; + var predictionEnd = aPredictionEnd; // Make sure we have the latest recommender config from the DB - the one // from the active recommenders list may be outdated @@ -1685,8 +1695,9 @@ private void computePredictions(LazyCas aOriginalCas, * @param aDataOwner * the annotation data owner */ - private void computePredictions(Predictions aPredictions, CAS aPredictionCas, - SourceDocument aDocument, String aDataOwner, int aPredictionBegin, int aPredictionEnd) + private void computePredictions(Predictions aActivePredictions, Predictions aPredictions, + CAS aPredictionCas, SourceDocument aDocument, String aDataOwner, int aPredictionBegin, + int aPredictionEnd) { var aSessionOwner = aPredictions.getSessionOwner(); @@ -1699,14 +1710,14 @@ private void computePredictions(Predictions aPredictions, CAS aPredictionCas, } LazyCas originalCas = new LazyCas(aDocument, aDataOwner); - for (EvaluatedRecommender r : recommenders) { - var layer = schemaService.getLayer(r.getRecommender().getLayer().getId()); + for (var recommender : recommenders) { + var layer = schemaService.getLayer(recommender.getRecommender().getLayer().getId()); if (!layer.isEnabled()) { continue; } - computePredictions(originalCas, r, aPredictions, aPredictionCas, aDocument, - aSessionOwner, aPredictionBegin, aPredictionEnd); + computePredictions(originalCas, recommender, aActivePredictions, aPredictions, + aPredictionCas, aDocument, aSessionOwner, aPredictionBegin, aPredictionEnd); } } catch (IOException e) { @@ -1727,20 +1738,23 @@ private void computePredictions(Predictions aPredictions, CAS aPredictionCas, public Predictions computePredictions(User aSessionOwner, Project aProject, List aDocuments, String aDataOwner, TaskMonitor aMonitor) { + var activePredictions = getPredictions(aSessionOwner, aProject); + var predictions = activePredictions != null ? new Predictions(activePredictions) + : new Predictions(aSessionOwner, aDataOwner, aProject); + try (var casHolder = new PredictionCasHolder()) { - Predictions predictions = new Predictions(aSessionOwner, aDataOwner, aProject); // Generate new predictions or inherit at the recommender level aMonitor.setMaxProgress(aDocuments.size()); for (SourceDocument document : aDocuments) { aMonitor.addMessage(LogMessage.info(this, "%s", document.getName())); aMonitor.incrementProgress(); - computePredictions(predictions, casHolder.cas, document, aDataOwner, -1, -1); + computePredictions(activePredictions, predictions, casHolder.cas, document, + aDataOwner, -1, -1); } return predictions; } catch (ResourceInitializationException e) { - Predictions predictions = new Predictions(aSessionOwner, aDataOwner, aProject); predictions.log( LogMessage.error(this, "Cannot create prediction CAS, stopping predictions!")); LOG.error("Cannot create prediction CAS, stopping predictions!"); @@ -1755,13 +1769,14 @@ public Predictions computePredictions(User aSessionOwner, Project aProject, { aMonitor.setMaxProgress(1); - var predictions = new Predictions(aSessionOwner, aDataOwner, aProject); var activePredictions = getPredictions(aSessionOwner, aProject); + var predictions = activePredictions != null ? new Predictions(activePredictions) + : new Predictions(aSessionOwner, aDataOwner, aProject); // Inherit at the document level. If inheritance at a recommender level is possible, // this is done below. if (activePredictions != null) { - for (SourceDocument document : aInherit) { + for (var document : aInherit) { inheritSuggestionsAtDocumentLevel(aProject, document, aSessionOwner, activePredictions, predictions); } @@ -1771,8 +1786,8 @@ public Predictions computePredictions(User aSessionOwner, Project aProject, final CAS predictionCas = casHolder.cas; // Generate new predictions or inherit at the recommender level - computePredictions(predictions, predictionCas, aCurrentDocument, aDataOwner, - aPredictionBegin, aPredictionEnd); + computePredictions(activePredictions, predictions, predictionCas, aCurrentDocument, + aDataOwner, aPredictionBegin, aPredictionEnd); } catch (ResourceInitializationException e) { predictions.log( @@ -1793,8 +1808,8 @@ private void inheritSuggestionsAtRecommenderLevel(Predictions predictions, CAS a Recommender aRecommender, Predictions activePredictions, SourceDocument document, User aUser) { - List suggestions = activePredictions - .getPredictionsByRecommenderAndDocument(aRecommender, document.getName()); + var suggestions = activePredictions.getPredictionsByRecommenderAndDocument(aRecommender, + document.getName()); if (suggestions.isEmpty()) { LOG.debug("{} for user {} on document {} in project {} there " // @@ -1826,13 +1841,11 @@ private void inheritSuggestionsAtDocumentLevel(Project aProject, SourceDocument return; } - List suggestions1 = aOldPredictions - .getPredictionsByDocument(aDocument.getName()); + var suggestions = aOldPredictions.getPredictionsByDocument(aDocument.getName()); LOG.debug("[{}]({}) for user [{}] on document {} in project {} inherited {} predictions", - "ALL", "--", aUser.getUsername(), aDocument, aProject, suggestions1.size()); + "ALL", "--", aUser.getUsername(), aDocument, aProject, suggestions.size()); - List suggestions = suggestions1; aNewPredictions.putPredictions(suggestions); aNewPredictions.markDocumentAsPredictionCompleted(aDocument); } @@ -1840,45 +1853,55 @@ private void inheritSuggestionsAtDocumentLevel(Project aProject, SourceDocument /** * Invokes the engine to produce new suggestions. */ - void generateSuggestions(Predictions aPredictions, RecommenderContext aCtx, + void generateSuggestions(Predictions aIncomingPredictions, RecommenderContext aCtx, RecommendationEngine aEngine, Predictions aActivePredictions, SourceDocument aDocument, CAS aOriginalCas, CAS aPredictionCas, int aPredictionBegin, int aPredictionEnd) throws RecommendationException { - var sessionOwner = aPredictions.getSessionOwner(); + var sessionOwner = aIncomingPredictions.getSessionOwner(); var recommender = aEngine.getRecommender(); - aPredictions.log(LogMessage.info(recommender.getName(), + // Perform the actual prediction + aIncomingPredictions.log(LogMessage.info(recommender.getName(), "Generating predictions for layer [%s]...", recommender.getLayer().getUiName())); LOG.trace("{}[{}]: Generating predictions for layer [{}]", sessionOwner, recommender.getName(), recommender.getLayer().getUiName()); - - // Perform the actual prediction var predictedRange = aEngine.predict(aCtx, aPredictionCas, aPredictionBegin, aPredictionEnd); // Extract the suggestions from the data which the recommender has written into the CAS - var suggestions = extractSuggestions(aOriginalCas, aPredictionCas, aDocument, recommender); + var generatedSuggestions = extractSuggestions(aIncomingPredictions.getGeneration(), + aOriginalCas, aPredictionCas, aDocument, recommender); + // Reconcile new suggestions with suggestions from previous run + var reconciliationResult = reconcile(aActivePredictions, aDocument, recommender, + predictedRange, generatedSuggestions); LOG.debug( - "{} for user {} on document {} in project {} generated {} predictions within range {}", - recommender, sessionOwner, aDocument, recommender.getProject(), suggestions.size(), - predictedRange); - aPredictions.log(LogMessage.info(recommender.getName(), // - "Generated [%d] predictions within range %s", suggestions.size(), predictedRange)); - - if (aActivePredictions != null) { - // Inherit annotations that are outside the range which was predicted. Note that the - // engine might actually predict a different range from what was requested. - List inheritableSuggestions = aActivePredictions + "{} for user {} on document {} in project {} generated {} predictions within range {} (+{}/-{}/={})", + recommender, sessionOwner, aDocument, recommender.getProject(), + generatedSuggestions.size(), predictedRange, reconciliationResult.added, + reconciliationResult.removed, reconciliationResult.aged); + aIncomingPredictions.log(LogMessage.info(recommender.getName(), // + "Generated [%d] predictions within range %s (+%d/-%d/=%d)", + generatedSuggestions.size(), predictedRange, reconciliationResult.added, + reconciliationResult.removed, reconciliationResult.aged)); + var suggestions = reconciliationResult.suggestions; + + // Inherit suggestions that are outside the range which was predicted. Note that the engine + // might actually predict a different range from what was requested. If the prediction + // covers the entire document, we can skip this. + if (aActivePredictions != null + && !predictedRange.equals(rangeCoveringDocument(aOriginalCas))) { + var inheritableSuggestions = aActivePredictions .getPredictionsByRecommenderAndDocument(recommender, aDocument.getName()) - .stream().filter(s -> !s.coveredBy(predictedRange)) // + .stream() // + .filter(s -> !s.coveredBy(predictedRange)) // .collect(toList()); LOG.debug("{} for user {} on document {} in project {} inherited {} " // + "predictions", recommender, sessionOwner, aDocument, recommender.getProject(), inheritableSuggestions.size()); - aPredictions.log(LogMessage.info(recommender.getName(), + aIncomingPredictions.log(LogMessage.info(recommender.getName(), "Inherited [%d] predictions from previous run", inheritableSuggestions.size())); suggestions.addAll(inheritableSuggestions); @@ -1888,14 +1911,70 @@ void generateSuggestions(Predictions aPredictions, RecommenderContext aCtx, // contains only the manually created annotations and *not* the suggestions. var groupedSuggestions = groupsOfType(SpanSuggestion.class, suggestions); calculateSpanSuggestionVisibility(sessionOwner.getUsername(), aDocument, aOriginalCas, - aPredictions.getDataOwner(), aEngine.getRecommender().getLayer(), + aIncomingPredictions.getDataOwner(), aEngine.getRecommender().getLayer(), groupedSuggestions, 0, aOriginalCas.getDocumentText().length()); + // FIXME calculateRelationSuggestionVisibility? + + aIncomingPredictions.putPredictions(suggestions); + } + + static ReconciliationResult reconcile(Predictions aActivePredictions, SourceDocument aDocument, + Recommender recommender, Range predictedRange, + List aNewProtoSuggesitons) + { + if (aActivePredictions == null) { + return new ReconciliationResult(aNewProtoSuggesitons.size(), 0, 0, + aNewProtoSuggesitons); + } + + var reconciledSuggestions = new LinkedHashSet(); + var addedSuggestions = new ArrayList(); + int agedSuggestionsCount = 0; + + var predictionsByRecommenderAndDocument = aActivePredictions + .getPredictionsByRecommenderAndDocument(recommender, aDocument.getName()); + + var existingSuggestionsByPosition = predictionsByRecommenderAndDocument.stream() // + .filter(s -> s.coveredBy(predictedRange)) // + .collect(groupingBy(AnnotationSuggestion::getPosition)); + + for (var newSuggestion : aNewProtoSuggesitons) { + var existingSuggestions = existingSuggestionsByPosition + .getOrDefault(newSuggestion.getPosition(), emptyList()).stream() // + .filter(s -> Objects.equals(s.getLabel(), newSuggestion.getLabel()) && // + s.getScore() == newSuggestion.getScore() && // + Objects.equals(s.getScoreExplanation(), + newSuggestion.getScoreExplanation())) + .collect(toList()); + + if (existingSuggestions.isEmpty()) { + addedSuggestions.add(newSuggestion); + reconciledSuggestions.add(newSuggestion); + continue; + } - aPredictions.putPredictions(suggestions); + if (existingSuggestions.size() > 1) { + LOG.debug("Recommender produced more than one suggestion with the same " + + "label, score and score explanation - reconciling with first one"); + } + + var existingSuggestion = existingSuggestions.get(0); + existingSuggestion.incrementAge(); + agedSuggestionsCount++; + reconciledSuggestions.add(existingSuggestion); + } + + var removedSuggestions = predictionsByRecommenderAndDocument.stream() // + .filter(s -> s.coveredBy(predictedRange)) // + .filter(s -> !reconciledSuggestions.contains(s)) // + .collect(toList()); + + return new ReconciliationResult(addedSuggestions.size(), removedSuggestions.size(), + agedSuggestionsCount, new ArrayList<>(reconciledSuggestions)); } - static List extractSuggestions(CAS aOriginalCas, CAS aPredictionCas, - SourceDocument aDocument, Recommender aRecommender) + static List extractSuggestions(int aGeneration, CAS aOriginalCas, + CAS aPredictionCas, SourceDocument aDocument, Recommender aRecommender) { var layer = aRecommender.getLayer(); var featureName = aRecommender.getFeature().getName(); @@ -1915,7 +1994,6 @@ static List extractSuggestions(CAS aOriginalCas, CAS aPred var isMultiLabels = TYPE_NAME_STRING_ARRAY.equals(labelFeature.getRange().getName()); var result = new ArrayList(); - int id = 0; var documentText = aOriginalCas.getDocumentText(); for (var predictedFS : aPredictionCas.select(predictedType)) { @@ -1943,7 +2021,8 @@ static List extractSuggestions(CAS aOriginalCas, CAS aPred for (var label : labels) { var suggestion = SpanSuggestion.builder() // - .withId(id) // + .withId(RelationSuggestion.NEW_ID) // + .withGeneration(aGeneration) // .withRecommender(aRecommender) // .withDocumentName(aDocument.getName()) // .withPosition(offsets) // @@ -1955,7 +2034,6 @@ static List extractSuggestions(CAS aOriginalCas, CAS aPred .withAutoAcceptMode(autoAcceptMode) // .build(); result.add(suggestion); - id++; } break; } @@ -1973,7 +2051,8 @@ static List extractSuggestions(CAS aOriginalCas, CAS aPred for (var label : labels) { var suggestion = RelationSuggestion.builder() // - .withId(id) // + .withId(RelationSuggestion.NEW_ID) // + .withGeneration(aGeneration) // .withRecommender(aRecommender) // .withDocumentName(aDocument.getName()) // .withPosition(position).withLabel(label) // @@ -1983,7 +2062,6 @@ static List extractSuggestions(CAS aOriginalCas, CAS aPred .withAutoAcceptMode(autoAcceptMode) // .build(); result.add(suggestion); - id++; } break; } @@ -2011,14 +2089,11 @@ private static AutoAcceptMode getAutoAcceptMode(FeatureStructure aFS, Feature aM private static String[] getPredictedLabels(FeatureStructure predictedFS, Feature predictedFeature, boolean isStringMultiValue) { - String[] labels; if (isStringMultiValue) { - labels = FSUtil.getFeature(predictedFS, predictedFeature, String[].class); - } - else { - labels = new String[] { predictedFS.getFeatureValueAsString(predictedFeature) }; + return FSUtil.getFeature(predictedFS, predictedFeature, String[].class); } - return labels; + + return new String[] { predictedFS.getFeatureValueAsString(predictedFeature) }; } /** @@ -2186,7 +2261,11 @@ public void calculateSpanSuggestionVisibility(String aSessionOwner, SourceDocume Collection> aRecommendations, int aWindowBegin, int aWindowEnd) { - Type type = getAnnotationType(aCas, aLayer); + LOG.trace( + "calculateSpanSuggestionVisibility() for layer {} on document {} in range [{}, {}]", + aLayer, aDocument, aWindowBegin, aWindowEnd); + + var type = getAnnotationType(aCas, aLayer); if (type == null) { // The type does not exist in the type system of the CAS. Probably it has not // been upgraded to the latest version of the type system yet. If this is the case, @@ -2222,7 +2301,7 @@ public void calculateSpanSuggestionVisibility(String aSessionOwner, SourceDocume // Reduce the suggestions to the ones for the given feature. We can use the tree here // since we only have a single SuggestionGroup for every position - Map> suggestions = new TreeMap<>( + var suggestions = new TreeMap>( comparingInt(Offset::getBegin).thenComparingInt(Offset::getEnd)); suggestionsInWindow.stream() .filter(group -> group.getFeature().equals(feature.getName())) // @@ -2232,8 +2311,8 @@ public void calculateSpanSuggestionVisibility(String aSessionOwner, SourceDocume }) // .forEach(group -> suggestions.put((Offset) group.getPosition(), group)); - hideSpanSuggestionsThatOverlapWithAnnotations(annotationsInWindow, suggestionsInWindow, - feature, feat, suggestions); + hideSpanSuggestionsThatOverlapWithAnnotations(annotationsInWindow, feature, feat, + suggestions); // Anything that was not hidden so far might still have been rejected suggestions.values().stream() // @@ -2245,9 +2324,8 @@ public void calculateSpanSuggestionVisibility(String aSessionOwner, SourceDocume } private void hideSpanSuggestionsThatOverlapWithAnnotations( - List annotationsInWindow, - List> suggestionsInWindow, AnnotationFeature feature, - Feature feat, Map> suggestions) + List annotationsInWindow, AnnotationFeature feature, Feature feat, + Map> suggestions) { // If there are no suggestions or annotations, there is nothing to do here if (annotationsInWindow.isEmpty() || suggestions.isEmpty()) { @@ -2257,12 +2335,12 @@ private void hideSpanSuggestionsThatOverlapWithAnnotations( // Reduce the annotations to the ones which have a non-null feature value. We need to // use a multi-valued map here because there may be multiple annotations at a // given position. - MultiValuedMap annotations = new ArrayListValuedHashMap<>(); + var annotations = new ArrayListValuedHashMap(); annotationsInWindow .forEach(fs -> annotations.put(new Offset(fs.getBegin(), fs.getEnd()), fs)); // We need to constructed a sorted list of the keys for the OverlapIterator below - List sortedAnnotationKeys = new ArrayList<>(annotations.keySet()); + var sortedAnnotationKeys = new ArrayList(annotations.keySet()); sortedAnnotationKeys.sort(comparingInt(Offset::getBegin).thenComparingInt(Offset::getEnd)); // This iterator gives us pairs of annotations and suggestions. Note that both lists @@ -2272,6 +2350,7 @@ private void hideSpanSuggestionsThatOverlapWithAnnotations( // Bulk-hide any groups that overlap with existing annotations on the current layer // and for the current feature + var hiddenForOverlap = new ArrayList(); while (oi.hasNext()) { if (oi.getA().overlaps(oi.getB())) { // Fetch the current suggestion and annotation @@ -2290,6 +2369,7 @@ private void hideSpanSuggestionsThatOverlapWithAnnotations( // or not. if (colocated) { suggestion.hide(FLAG_OVERLAP); + hiddenForOverlap.add(suggestion); continue; } @@ -2297,6 +2377,7 @@ private void hideSpanSuggestionsThatOverlapWithAnnotations( // annotation with no label, but only if the offsets differ if (feature.getLayer().isAllowStacking() && !colocated) { suggestion.hide(FLAG_OVERLAP); + hiddenForOverlap.add(suggestion); continue; } } @@ -2315,6 +2396,7 @@ private void hideSpanSuggestionsThatOverlapWithAnnotations( // at the same position then we hide if (label != null && label.equals(suggestion.getLabel()) && colocated) { suggestion.hide(FLAG_OVERLAP); + hiddenForOverlap.add(suggestion); continue; } @@ -2322,6 +2404,7 @@ private void hideSpanSuggestionsThatOverlapWithAnnotations( // stacking is not enabled - then we need to hide if (!feature.getLayer().isAllowStacking()) { suggestion.hide(FLAG_OVERLAP); + hiddenForOverlap.add(suggestion); continue; } } @@ -2335,6 +2418,13 @@ private void hideSpanSuggestionsThatOverlapWithAnnotations( oi.step(); } + + if (LOG.isTraceEnabled()) { + LOG.trace("Hidden due to overlapping: {}", hiddenForOverlap.size()); + for (var s : hiddenForOverlap) { + LOG.trace("- {}", s); + } + } } @Override @@ -2486,8 +2576,12 @@ CAS cloneAndMonkeyPatchCAS(Project aProject, CAS aSourceCas, CAS aTargetCas) for (var feature : features) { var td = tsd.getType(feature.getLayer().getName()); if (td == null) { - LOG.trace("Could not monkey patch feature {} because type for layer {} was not " - + "found in the type system", feature, feature.getLayer()); + if (!WebAnnoConst.CHAIN_TYPE.equals(feature.getLayer().getType())) { + LOG.trace( + "Could not monkey patch feature {} because type for layer {} was not " + + "found in the type system", + feature, feature.getLayer()); + } continue; } @@ -2917,8 +3011,9 @@ public void deleteSkippedSuggestions(String aSessionOwner, User aDataOwner, { var state = getState(aSessionOwner, aLayer.getProject()); synchronized (state) { - state.learningRecords.getOrDefault(aLayer, Collections.emptyList()).removeIf( - r -> Objects.equals(r.getUser(), aDataOwner) && r.getUserAction() == SKIPPED); + state.learningRecords.getOrDefault(aLayer, Collections.emptyList()) + .removeIf(r -> Objects.equals(r.getUser(), aDataOwner.getUsername()) + && r.getUserAction() == SKIPPED); } String sql = String.join("\n", // @@ -2960,7 +3055,6 @@ public CAS get() throws IOException private static class PredictionCasHolder implements AutoCloseable { - private final CAS cas; public PredictionCasHolder() throws ResourceInitializationException @@ -2975,4 +3069,9 @@ public void close() CasStorageSession.get().remove(cas); } } + + final record ReconciliationResult(int added, int removed, int aged, + List suggestions) + { + } } diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/LogDialog.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/LogDialog.java index fd619d98dc4..6ccbb2fded1 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/LogDialog.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/LogDialog.java @@ -66,7 +66,7 @@ public void show(AjaxRequestTarget aTarget) model = new ListModel<>(asList(group)); } - LogDialogContent content = new LogDialogContent(ModalDialog.CONTENT_ID, model); + var content = new LogDialogContent(ModalDialog.CONTENT_ID, model); open(content, aTarget); aTarget.focusComponent(content.getFocusComponent()); } diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java index f471f1df7f2..3202793a85c 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java @@ -106,7 +106,7 @@ public RecommenderInfoPanel(String aId, IModel aModel) + repeat(" ", p.getTodo()); })).setEscapeModelStrings(false)); // SAFE - RENDERING ONLY SPECIFIC ICONS - WebMarkupContainer recommenderContainer = new WebMarkupContainer("recommenderContainer"); + var recommenderContainer = new WebMarkupContainer("recommenderContainer"); add(recommenderContainer); ListView searchResultGroups = new ListView("recommender") @@ -116,12 +116,12 @@ public RecommenderInfoPanel(String aId, IModel aModel) @Override protected void populateItem(ListItem item) { - Recommender recommender = item.getModelObject(); + var recommender = item.getModelObject(); Optional evaluatedRecommender = recommendationService .getEvaluatedRecommender(sessionOwner, recommender); item.add(new Label("name", recommender.getName())); - WebMarkupContainer state = new WebMarkupContainer("state"); + var state = new WebMarkupContainer("state"); if (evaluatedRecommender.isPresent()) { EvaluatedRecommender evalRec = evaluatedRecommender.get(); if (evalRec.isActive()) { @@ -147,7 +147,7 @@ protected void populateItem(ListItem item) Optional evalResult = evaluatedRecommender .map(EvaluatedRecommender::getEvaluationResult); - WebMarkupContainer resultsContainer = new WebMarkupContainer("resultsContainer"); + var resultsContainer = new WebMarkupContainer("resultsContainer"); // Show results only if the evaluation was not skipped (and of course only if the // result is actually present). resultsContainer.setVisible(evalResult.map(r -> !r.isEvaluationSkipped()) @@ -189,10 +189,11 @@ protected void populateItem(ListItem item) .isModelExportSupported())); item.add(exportModel); - item.add(new Label("noEvaluationMessage", - evaluatedRecommender.map(EvaluatedRecommender::getReasonForState) - .orElse("Waiting for evalation...")) - .add(visibleWhen(() -> !resultsContainer.isVisible()))); + item.add(new Label("noEvaluationMessage", evaluatedRecommender // + .flatMap(r -> r.getEvaluationResult().getErrorMsg()) + // .map(EvaluatedRecommender::getReasonForState) + .orElse("Waiting for evalation...")) + .add(visibleWhen(() -> !resultsContainer.isVisible()))); } }; IModel> recommenders = LoadableDetachableModel diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java index daabdb12d04..5506258ff2d 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java @@ -46,12 +46,10 @@ import de.tudarmstadt.ukp.clarin.webanno.support.logging.LogMessage; import de.tudarmstadt.ukp.inception.annotation.storage.CasStorageSession; import de.tudarmstadt.ukp.inception.recommendation.api.RecommendationService; -import de.tudarmstadt.ukp.inception.recommendation.api.evaluation.DataSplitter; import de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult; import de.tudarmstadt.ukp.inception.recommendation.api.evaluation.PercentageBasedSplitter; import de.tudarmstadt.ukp.inception.recommendation.api.model.EvaluatedRecommender; import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderEvaluationResultEvent; import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderTaskNotificationEvent; @@ -60,8 +58,11 @@ /** * This task evaluates all available classification tools for all annotation layers of the current - * project. If a classifier exceeds its specific activation f-score limit during the evaluation it - * is selected for active prediction. + * project. If a classifier exceeds its specific activation score limit during the evaluation it is + * selected for active prediction. + * + * If the threshold is 0 (or less), the evaluation should be considered optional. That is, if the + * evaluation fails (e.g. because of too little data), then the training should still be scheduled. */ public class SelectionTask extends RecommendationTask_ImplBase @@ -301,17 +302,23 @@ private Optional evaluate(User user, Recommender recommend } log.info("[{}][{}]: Evaluating...", userName, recommender.getName()); - DataSplitter splitter = new PercentageBasedSplitter(0.8, 10); - RecommendationEngine recommendationEngine = factory.build(recommender); + var splitter = new PercentageBasedSplitter(0.8, 10); + var recommendationEngine = factory.build(recommender); - EvaluationResult result = recommendationEngine.evaluate(aCasses.get(), splitter); + var result = recommendationEngine.evaluate(aCasses.get(), splitter); + double threshold = recommender.getThreshold(); if (result.isEvaluationSkipped()) { + var evaluationIsOptional = recommender.getThreshold() <= 0.0d; + if (evaluationIsOptional) { + return Optional.of( + activateRecommenderAboveThreshold(user, recommender, result, 0, threshold)); + } + return Optional.of(skipRecommenderDueToFailedEvaluation(user, recommender, result)); } double score = result.computeF1Score(); - double threshold = recommender.getThreshold(); if (score >= threshold) { return Optional.of( activateRecommenderAboveThreshold(user, recommender, result, score, threshold)); diff --git a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java index 632e1d48faf..1b2bc655bca 100644 --- a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java +++ b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplIntegrationTest.java @@ -39,7 +39,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import java.util.List; import java.util.Optional; import org.apache.uima.cas.CAS; @@ -146,10 +145,11 @@ public void listRecommenders_WithOneEnabledRecommender_ShouldReturnStoredRecomme { sut.createOrUpdateRecommender(rec); - List enabledRecommenders = sut.listEnabledRecommenders(rec.getLayer()); + var enabledRecommenders = sut.listEnabledRecommenders(rec.getLayer()); - assertThat(enabledRecommenders).as("Check that the previously created recommender is found") - .hasSize(1).contains(rec); + assertThat(enabledRecommenders) // + .as("Check that the previously created recommender is found") // + .containsExactly(rec); } @SuppressWarnings("unchecked") @@ -174,16 +174,13 @@ public void getNumOfEnabledRecommenders_WithNoEnabledRecommender() rec.setEnabled(false); testEntityManager.persist(rec); - long numOfRecommenders = sut.countEnabledRecommenders(); - assertThat(numOfRecommenders).isEqualTo(0); + assertThat(sut.countEnabledRecommenders()).isEqualTo(0); } @Test public void getRecommenders_WithOneEnabledRecommender_ShouldReturnStoredRecommender() { - Optional enabledRecommenders = sut.getEnabledRecommender(rec.getId()); - - assertThat(enabledRecommenders) + assertThat(sut.getEnabledRecommender(rec.getId())) .as("Check that only the previously created recommender is found").isPresent() .contains(rec); } @@ -194,15 +191,14 @@ public void getRecommenders_WithOnlyDisabledRecommender_ShouldReturnEmptyList() rec.setEnabled(false); testEntityManager.persist(rec); - Optional enabledRecommenders = sut.getEnabledRecommender(rec.getId()); - - assertThat(enabledRecommenders).as("Check that no recommender is found").isEmpty(); + assertThat(sut.getEnabledRecommender(rec.getId())) // + .as("Check that no recommender is found") // + .isEmpty(); } @Test public void getRecommenders_WithOtherRecommenderId_ShouldReturnEmptyList() { - long otherId = 9999L; Optional enabledRecommenders = sut.getEnabledRecommender(otherId); diff --git a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java index 2aa41efbb4a..5fad7e40318 100644 --- a/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java +++ b/inception/inception-recommendation/src/test/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImplTest.java @@ -43,6 +43,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; +import java.util.Arrays; + import org.apache.uima.UIMAFramework; import org.apache.uima.fit.factory.CasFactory; import org.apache.uima.fit.factory.JCasFactory; @@ -53,15 +55,19 @@ import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature; import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; +import de.tudarmstadt.ukp.clarin.webanno.model.Project; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.clarin.webanno.security.model.User; import de.tudarmstadt.ukp.clarin.webanno.support.WebAnnoConst; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; import de.tudarmstadt.ukp.inception.recommendation.api.model.AnnotationSuggestion; import de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecord; import de.tudarmstadt.ukp.inception.recommendation.api.model.Offset; +import de.tudarmstadt.ukp.inception.recommendation.api.model.Predictions; import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; import de.tudarmstadt.ukp.inception.recommendation.api.model.SpanSuggestion; +import de.tudarmstadt.ukp.inception.rendering.model.Range; class RecommendationServiceImplTest { @@ -87,6 +93,60 @@ void setup() feature2 = AnnotationFeature.builder().withName("feat2").build(); } + @Test + void testReconciliation() throws Exception + { + var sessionOwner = User.builder().withUsername("user").build(); + var doc = SourceDocument.builder().withName("doc1").build(); + var layer = AnnotationLayer.builder().withId(1l).build(); + var feature = AnnotationFeature.builder().withName("feature").withLayer(layer).build(); + var rec = Recommender.builder().withId(1l).withName("rec").withLayer(layer) + .withFeature(feature).build(); + var project = Project.builder().withId(1l).build(); + + var existingSuggestions = Arrays. asList( // + SpanSuggestion.builder() // + .withId(0) // + .withPosition(new Offset(0, 10)) // + .withDocumentName(doc.getName()) // + .withLabel("aged") // + .withRecommender(rec) // + .build(), + SpanSuggestion.builder() // + .withId(1) // + .withPosition(new Offset(0, 10)) // + .withDocumentName(doc.getName()) // + .withLabel("removed") // + .withRecommender(rec) // + .build()); + var activePredictions = new Predictions(sessionOwner, sessionOwner.getUsername(), project); + activePredictions.putPredictions(existingSuggestions); + + var newSuggestions = Arrays. asList( // + SpanSuggestion.builder() // + .withId(2) // + .withPosition(new Offset(0, 10)) // + .withDocumentName(doc.getName()) // + .withLabel("aged") // + .withRecommender(rec) // + .build(), + SpanSuggestion.builder() // + .withId(3) // + .withPosition(new Offset(0, 10)) // + .withDocumentName(doc.getName()) // + .withLabel("added") // + .withRecommender(rec) // + .build()); + + var result = RecommendationServiceImpl.reconcile(activePredictions, doc, rec, + new Range(0, 10), newSuggestions); + + assertThat(result.suggestions()) // + .extracting(AnnotationSuggestion::getId, AnnotationSuggestion::getLabel, + AnnotationSuggestion::getAge) // + .containsExactlyInAnyOrder(tuple(0, "aged", 1), tuple(3, "added", 0)); + } + @Test void thatRejectedSuggestionIsHidden() { @@ -325,7 +385,7 @@ void testExtractSuggestionsWithSpanSuggestions() throws Exception .withFeature(FEATURE_NAME_IS_PREDICTION, true) // .buildAndAddToIndexes(); - var suggestions = extractSuggestions(targetCas, suggestionCas, doc1, recommender); + var suggestions = extractSuggestions(0, targetCas, suggestionCas, doc1, recommender); assertThat(suggestions) // .extracting( // diff --git a/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationDetailEditorPanel.java b/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationDetailEditorPanel.java index bb97e38facc..09eb537f3c2 100644 --- a/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationDetailEditorPanel.java +++ b/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationDetailEditorPanel.java @@ -195,15 +195,14 @@ public void renderHead(IHeaderResponse aResponse) private LambdaAjaxLink createNextAnnotationButton() { - LambdaAjaxLink link = new LambdaAjaxLink("nextAnnotation", this::actionNextAnnotation); + var link = new LambdaAjaxLink("nextAnnotation", this::actionNextAnnotation); link.add(new InputBehavior(new KeyType[] { Shift, Right }, click)); return link; } private LambdaAjaxLink createPreviousAnnotationButton() { - LambdaAjaxLink link = new LambdaAjaxLink("previousAnnotation", - this::actionPreviousAnnotation); + var link = new LambdaAjaxLink("previousAnnotation", this::actionPreviousAnnotation); link.add(new InputBehavior(new KeyType[] { Shift, Left }, click)); return link; } @@ -1042,13 +1041,13 @@ private void loadFeatureEditorModels(AjaxRequestTarget aTarget) { LOG.trace("loadFeatureEditorModels()"); - CAS aCas = getEditorCas(); + var aCas = getEditorCas(); - AnnotatorState state = getModelObject(); - Selection selection = state.getSelection(); + var state = getModelObject(); + var selection = state.getSelection(); - List featureStates = state.getFeatureStates(); - for (FeatureState featureState : featureStates) { + var featureStates = state.getFeatureStates(); + for (var featureState : featureStates) { if (StringUtils.isNotBlank(featureState.feature.getLinkTypeName())) { featureState.value = new ArrayList<>(); } diff --git a/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationInfoPanel.java b/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationInfoPanel.java index c709aca7223..c5a7beaa0d1 100644 --- a/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationInfoPanel.java +++ b/inception/inception-ui-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/annotation/detail/AnnotationInfoPanel.java @@ -23,6 +23,7 @@ import java.lang.invoke.MethodHandles; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.wicket.markup.html.WebMarkupContainer; import org.apache.wicket.markup.html.basic.Label; import org.apache.wicket.markup.html.panel.Panel; @@ -104,14 +105,13 @@ private Label createSelectedAnnotationTypeLabel() { Label label = new Label("selectedAnnotationType", LoadableDetachableModel.of(() -> { try { - AnnotationDetailEditorPanel editorPanel = findParent( - AnnotationDetailEditorPanel.class); + var editorPanel = findParent(AnnotationDetailEditorPanel.class); return String.valueOf(selectFsByAddr(editorPanel.getEditorCas(), getModelObject().getSelection().getAnnotation().getId())).trim(); } catch (Exception e) { LOG.warn("Unable to render selected annotation type", e); - return ""; + return ExceptionUtils.getRootCauseMessage(e); } })); label.setOutputMarkupPlaceholderTag(true); diff --git a/inception/pom.xml b/inception/pom.xml index 8e54ef05aee..15ff48787e6 100644 --- a/inception/pom.xml +++ b/inception/pom.xml @@ -28,7 +28,7 @@ 6g - 11 + 17 ${maven.compiler.release} ${maven.compiler.release} yyyy-MM-dd HH:mm