#4995 - Ability to select states of documents to include in bulk recommender processing

inception/inception-processing/pom.xml

@@ -106,6 +106,10 @@
       <groupId>org.apache.wicket</groupId>
       <artifactId>wicket-spring</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.wicket</groupId>
+      <artifactId>wicket-util</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.wicket</groupId>
       <artifactId>wicket-request</artifactId>

inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/AnnotationDocumentStatesChoice.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Technische Universität Darmstadt under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The Technische Universität Darmstadt 
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.
+ *  
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package de.tudarmstadt.ukp.inception.processing.recommender;
+
+import static org.apache.wicket.markup.html.form.AbstractChoice.LabelPosition.AFTER;
+
+import java.util.Collection;
+
+import org.apache.wicket.markup.html.form.CheckBoxMultipleChoice;
+import org.apache.wicket.markup.html.form.EnumChoiceRenderer;
+import org.apache.wicket.model.IModel;
+import org.apache.wicket.util.value.AttributeMap;
+import org.apache.wicket.util.value.IValueMap;
+
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState;
+
+public class AnnotationDocumentStatesChoice
+    extends CheckBoxMultipleChoice<AnnotationDocumentState>
+{
+    private static final long serialVersionUID = 7627977162174971025L;
+
+    public AnnotationDocumentStatesChoice(String aId)
+    {
+        this(aId, null);
+    }
+
+    public AnnotationDocumentStatesChoice(String aId,
+            IModel<Collection<AnnotationDocumentState>> aModel)
+    {
+        super(aId);
+        setModel(aModel);
+        setPrefix("<div class=\"form-check form-switch\">");
+        setSuffix("</div>");
+        setLabelPosition(AFTER);
+        setChoiceRenderer(new EnumChoiceRenderer<>(this));
+    }
+
+    @Override
+    protected IValueMap getAdditionalAttributesForLabel(int aIndex, AnnotationDocumentState aChoice)
+    {
+        var attributes = new AttributeMap();
+        attributes.put("class", "form-check-label");
+        return attributes;
+    }
+
+    @Override
+    protected IValueMap getAdditionalAttributes(int aIndex, AnnotationDocumentState aChoice)
+    {
+        var attributes = new AttributeMap();
+        attributes.put("class", "form-check-input");
+        return attributes;
+    }
+}

inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkPredictionTask.java

@@ -21,19 +21,22 @@
 import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasUpgradeMode.AUTO_CAS_UPGRADE;
 import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState.FINISHED;
 import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState.IN_PROGRESS;
-import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState.NEW;
 import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentStateChangeFlag.EXPLICIT_ANNOTATOR_USER_ACTION;
 import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordChangeLocation.AUTO_ACCEPT;
 import static de.tudarmstadt.ukp.inception.recommendation.api.model.LearningRecordUserAction.ACCEPTED;
 import static de.tudarmstadt.ukp.inception.scheduling.TaskScope.PROJECT;
+import static de.tudarmstadt.ukp.inception.scheduling.TaskState.CANCELLED;
 
 import java.io.IOException;
 import java.io.Serializable;
 import java.lang.invoke.MethodHandles;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 
 import org.apache.commons.lang3.Validate;
 import org.apache.uima.cas.AnnotationBaseFS;
@@ -42,6 +45,8 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocument;
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState;
 import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature;
 import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
 import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;
@@ -56,7 +61,6 @@
 import de.tudarmstadt.ukp.inception.recommendation.tasks.RecommendationTask_ImplBase;
 import de.tudarmstadt.ukp.inception.scheduling.ProjectTask;
 import de.tudarmstadt.ukp.inception.scheduling.SchedulingService;
-import de.tudarmstadt.ukp.inception.scheduling.TaskState;
 import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService;
 import de.tudarmstadt.ukp.inception.schema.api.adapter.AnnotationException;
 import de.tudarmstadt.ukp.inception.support.WebAnnoConst;
@@ -75,7 +79,8 @@ public class BulkPredictionTask
     private final String dataOwner;
     private final Recommender recommender;
     private final Map<AnnotationFeature, Serializable> processingMetadata;
-    private boolean finishDocumentsWithoutRecommendations;
+    private final boolean finishDocumentsWithoutRecommendations;
+    private final Set<AnnotationDocumentState> statesToProcess;
 
     private @Autowired UserDao userService;
     private @Autowired DocumentService documentService;
@@ -89,8 +94,9 @@ public BulkPredictionTask(Builder<? extends Builder<?>> aBuilder)
 
         recommender = aBuilder.recommender;
         dataOwner = aBuilder.dataOwner;
-        processingMetadata = aBuilder.processingMetadata;
         finishDocumentsWithoutRecommendations = aBuilder.finishDocumentsWithoutRecommendations;
+        processingMetadata = new HashMap<>(aBuilder.processingMetadata);
+        statesToProcess = new HashSet<>(aBuilder.statesToProcess);
     }
 
     @Override
@@ -105,38 +111,39 @@ public void execute()
     {
         var dataOwnerUser = userService.get(dataOwner);
         var monitor = getMonitor();
+        var visitedDocuments = new HashSet<SourceDocument>();
         var processedDocumentsCount = 0;
         var annotationsCount = 0;
         var suggestionsCount = 0;
-        int maxProgress;
+        var maxProgress = 0;
 
         while (true) {
-            // Find all documents currently in the document (which may have changed since the last
+            // Find all documents currently in the project (which may have changed since the last
             // iteration)
             var annotatableDocuments = documentService.listAnnotatableDocuments(getProject(),
                     dataOwnerUser);
 
-            // Find all documents that still need processing (i.e. which are in state NEW explicitly
-            // or implicitly).
-            var processableDocuments = annotatableDocuments.entrySet().stream() //
-                    .filter(e -> e.getValue() == null || e.getValue().getState() == NEW) //
+            var documentsToProcess = annotatableDocuments.entrySet().stream() //
+                    .filter(e -> isInProcessableState(e.getKey(), e.getValue())) //
+                    .filter(e -> !visitedDocuments.contains(e.getKey())) //
                     .map(e -> e.getKey()) //
                     .toList();
 
             maxProgress = annotatableDocuments.size();
-            var progress = maxProgress - processableDocuments.size();
-            if (processableDocuments.isEmpty() || monitor.isCancelled()) {
+            var progress = maxProgress - documentsToProcess.size();
+            if (documentsToProcess.isEmpty() || monitor.isCancelled()) {
                 monitor.setProgressWithMessage(progress, maxProgress,
                         LogMessage.info(this,
                                 "%d annotations generated from %d suggestions in %d documents",
                                 annotationsCount, suggestionsCount, processedDocumentsCount));
                 if (monitor.isCancelled()) {
-                    monitor.setState(TaskState.CANCELLED);
+                    monitor.setState(CANCELLED);
                 }
                 break;
             }
 
-            var doc = processableDocuments.get(0);
+            var doc = documentsToProcess.get(0);
+            visitedDocuments.add(doc);
             var annDoc = documentService.createOrGetAnnotationDocument(doc, dataOwnerUser);
 
             monitor.setProgressWithMessage(progress, maxProgress,
@@ -177,7 +184,18 @@ public void execute()
         }
 
         monitor.setProgressWithMessage(processedDocumentsCount, maxProgress,
-                LogMessage.info(this, "Prediction complete"));
+                LogMessage.info(this,
+                        "%d annotations generated from %d suggestions in %d documents",
+                        annotationsCount, suggestionsCount, processedDocumentsCount));
+    }
+
+    private boolean isInProcessableState(SourceDocument aSourceDocument,
+            AnnotationDocument aAnnotationDocument)
+    {
+        var effectiveState = aAnnotationDocument == null //
+                ? AnnotationDocumentState.NEW //
+                : aAnnotationDocument.getState();
+        return statesToProcess.contains(effectiveState);
     }
 
     private void addProcessingMetadataAnnotation(SourceDocument doc, CAS cas)
@@ -265,6 +283,7 @@ public static class Builder<T extends Builder<?>>
         private String dataOwner;
         private Map<AnnotationFeature, Serializable> processingMetadata;
         private boolean finishDocumentsWithoutRecommendations;
+        private final Set<AnnotationDocumentState> statesToProcess = new HashSet<>();
 
         @SuppressWarnings("unchecked")
         public T withRecommender(Recommender aRecommender)
@@ -280,6 +299,17 @@ public T withProcessingMetadata(Map<AnnotationFeature, Serializable> aProcessing
             return (T) this;
         }
 
+        @SuppressWarnings("unchecked")
+        public T withStatesToProcess(Collection<AnnotationDocumentState> aStates)
+        {
+            statesToProcess.clear();
+
+            if (aStates != null) {
+                statesToProcess.addAll(aStates);
+            }
+            return (T) this;
+        }
+
         /**
          * @param aDataOwner
          *            the user owning the annotations currently shown in the editor (this can differ

inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkRecommenderPanel.html

@@ -27,18 +27,23 @@
         <wicket:message key="applyRecommenderDescription"/>
       </p>
       <div class="container">
+        <div class="row form-row form-floating" wicket:enclosure="recommender">
+          <select wicket:id="recommender" class="form-select" wicket:message="placeholder:recommender"/>
+          <label wicket:for="recommender">
+            <wicket:label key="recommender"/>
+          </label>
+        </div>
+
         <div class="row form-row form-floating" wicket:enclosure="user">
           <select wicket:id="user" class="form-select" wicket:message="placeholder:user"/>
           <label wicket:for="user">
             <wicket:label key="user"/>
           </label>
         </div>
-
-        <div class="row form-row form-floating" wicket:enclosure="recommender">
-          <select wicket:id="recommender" class="form-select" wicket:message="placeholder:recommender"/>
-          <label wicket:for="recommender">
-            <wicket:label key="recommender"/>
-          </label>
+
+        <div class="row form-row border rounded" wicket:enclosure="states">
+          <div class="form-text"><wicket:message key="states"/></div>
+          <div wicket:id="states"/>
         </div>
 
         <div class="row form-row" wicket:enclosure="finishDocumentsWithoutRecommendations">

inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkRecommenderPanel.java

@@ -17,9 +17,12 @@
  */
 package de.tudarmstadt.ukp.inception.processing.recommender;
 
+import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState.IN_PROGRESS;
+import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState.NEW;
 import static de.tudarmstadt.ukp.clarin.webanno.model.PermissionLevel.ANNOTATOR;
 import static de.tudarmstadt.ukp.inception.support.lambda.HtmlElementEvents.CHANGE_EVENT;
 import static de.tudarmstadt.ukp.inception.support.lambda.LambdaBehavior.visibleWhenNot;
+import static java.util.Arrays.asList;
 
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -38,6 +41,7 @@
 import org.apache.wicket.model.LoadableDetachableModel;
 import org.apache.wicket.spring.injection.annot.SpringBean;
 
+import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState;
 import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature;
 import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
 import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer_;
@@ -91,6 +95,9 @@ public BulkRecommenderPanel(String aId, IModel<Project> aModel)
                 .setChoiceRenderer(new ChoiceRenderer<>(Recommender_.NAME)) //
                 .setRequired(true));
 
+        queue(new AnnotationDocumentStatesChoice("states") //
+                .setChoices(asList(NEW, IN_PROGRESS)));
+
         processingMetadata = new FeatureEditorPanel("processingMetadata");
         processingMetadata.setOutputMarkupPlaceholderTag(true);
         queue(processingMetadata);
@@ -126,6 +133,7 @@ private void actionStartProcessing(AjaxRequestTarget aTarget, Form<FormData> aFo
                 .withTrigger("User request") //
                 .withDataOwner(formData.user.getUsername()) //
                 .withProcessingMetadata(metadata) //
+                .withStatesToProcess(formData.states) //
                 .withFinishDocumentsWithoutRecommendations(
                         formData.finishDocumentsWithoutRecommendations) //
                 .build());
@@ -204,5 +212,11 @@ private static class FormData
         private Recommender recommender;
         private AnnotationLayer processingMetadataLayer;
         private boolean finishDocumentsWithoutRecommendations;
+        private List<AnnotationDocumentState> states;
+
+        {
+            states = new ArrayList<>();
+            states.add(NEW);
+        }
     }
 }

inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkRecommenderPanel.properties

@@ -15,9 +15,11 @@
 # limitations under the License.
 applyRecommender=Process documents using recommender
 startProcessing=Start processing...
-applyRecommenderDescription=Apply a recommender to all documents the chosen annotator has <i>not started</i> working \
-  on yet. Once the recommender has been applied, the documents are marked as <i>finished</i>. Only recommenders that \
-  do not require training can be used here.
+applyRecommenderDescription=Apply a recommender to all documents in the chosen states. \
+  Once the recommender has been applied, the documents are marked as <i>finished</i>. \
+  Only recommenders that do not require training can be used here.
 processingMetadataLayer=Processing metadata layer
 finishDocumentsWithoutRecommendations=Mark a document as finished even if no annotations were generated in it
 processingMetadataLayer.nullValid=- None -
+states=Process all annotation documents in these states
+user=Target user

inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/recommender/DefaultTrainableRecommenderTraitsEditor.properties

@@ -15,8 +15,3 @@
 # limitations under the License.
 
 statesForTraining=States used for training
-
-AnnotationDocumentState.NEW=Annotation not started yet (new)
-AnnotationDocumentState.IN_PROGRESS=Annotation in progress
-AnnotationDocumentState.FINISHED=Annotation finished
-AnnotationDocumentState.IGNORE=Document not available for annotation (locked)

inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/recommender/TrainingStatesChoice.java

@@ -43,7 +43,7 @@ public TrainingStatesChoice(String aId, IModel<Recommender> recommenderModel)
 
         // We need to invert the states in documentStates, as the recommender stores the
         // ones to ignore, not the ones to consider
-        IModel<Set<AnnotationDocumentState>> model = new IModel<Set<AnnotationDocumentState>>()
+        var model = new IModel<Set<AnnotationDocumentState>>()
         {
             private static final long serialVersionUID = 2894838629097952859L;
 
@@ -59,15 +59,14 @@ public void setObject(Set<AnnotationDocumentState> states)
             @Override
             public Set<AnnotationDocumentState> getObject()
             {
-                Set<AnnotationDocumentState> ignoredStates = recommenderModel.getObject()
-                        .getStatesIgnoredForTraining();
+                var ignoredStates = recommenderModel.getObject().getStatesIgnoredForTraining();
 
                 return invert(ignoredStates);
             }
 
             private Set<AnnotationDocumentState> invert(Set<AnnotationDocumentState> states)
             {
-                Set<AnnotationDocumentState> result = getAllPossibleDocumentStates();
+                var result = getAllPossibleDocumentStates();
 
                 if (states == null) {
                     return result;
@@ -88,15 +87,15 @@ private Set<AnnotationDocumentState> invert(Set<AnnotationDocumentState> states)
     @Override
     protected IValueMap getAdditionalAttributesForLabel(int aIndex, AnnotationDocumentState aChoice)
     {
-        AttributeMap attributes = new AttributeMap();
+        var attributes = new AttributeMap();
         attributes.put("class", "form-check-label");
         return attributes;
     }
 
     @Override
     protected IValueMap getAdditionalAttributes(int aIndex, AnnotationDocumentState aChoice)
     {
-        AttributeMap attributes = new AttributeMap();
+        var attributes = new AttributeMap();
         attributes.put("class", "form-check-input");
         return attributes;
     }