From 474871c3877dd7403f3277a25373003000504b57 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 23 Oct 2023 22:39:27 +0200 Subject: [PATCH] #4139 - Reusable popover component for annotation editors - Send sentence VID to brat - Use the new popover also for the sentence IDs in the brat editor - Clean up listeners in popover component - Add loading indicator to popover component --- .../SentenceOrientedPagingStrategy.java | 4 +- .../ukp/inception/rendering/paging/Unit.java | 13 +- .../brat/render/BratSerializerImpl.java | 7 +- .../brat/render/model/SentenceComment.java | 15 +- .../src/main/ts/src/protocol/Protocol.ts | 2 +- .../src/main/ts/src/visualizer/Comment.ts | 3 + .../src/main/ts/src/visualizer/Visualizer.ts | 11 +- .../main/ts/src/visualizer_ui/VisualizerUI.ts | 2 +- inception/inception-diam/pom.xml | 9 +- .../LazyDetailsLookupServiceImpl.java | 49 +++-- .../src/widget/AnnotationDetailPopOver.svelte | 169 +++++++++++------- 11 files changed, 189 insertions(+), 95 deletions(-) diff --git a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/paging/SentenceOrientedPagingStrategy.java b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/paging/SentenceOrientedPagingStrategy.java index e34a51b5ff9..024507bf96a 100644 --- a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/paging/SentenceOrientedPagingStrategy.java +++ b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/paging/SentenceOrientedPagingStrategy.java @@ -35,6 +35,7 @@ import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.inception.rendering.editorstate.AnnotatorState; import de.tudarmstadt.ukp.inception.rendering.paging.Unit; +import de.tudarmstadt.ukp.inception.rendering.vmodel.VID; public class SentenceOrientedPagingStrategy extends PagingStrategy_ImplBase @@ -78,7 +79,8 @@ private Unit toUnit(int aIndex, AnnotationFS aSentence) catch (IllegalArgumentException e) { // Ignore if there is no "id" feature on the sentence } - return new Unit(sentId, aIndex, aSentence.getBegin(), aSentence.getEnd()); + return new Unit(VID.of(aSentence), sentId, aIndex, aSentence.getBegin(), + aSentence.getEnd()); } @Override diff --git a/inception/inception-api-render/src/main/java/de/tudarmstadt/ukp/inception/rendering/paging/Unit.java b/inception/inception-api-render/src/main/java/de/tudarmstadt/ukp/inception/rendering/paging/Unit.java index b54eef7be04..3c4edd6189d 100644 --- a/inception/inception-api-render/src/main/java/de/tudarmstadt/ukp/inception/rendering/paging/Unit.java +++ b/inception/inception-api-render/src/main/java/de/tudarmstadt/ukp/inception/rendering/paging/Unit.java @@ -24,6 +24,8 @@ import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; +import de.tudarmstadt.ukp.inception.rendering.vmodel.VID; + public class Unit implements Serializable { @@ -32,6 +34,7 @@ public class Unit private final int index; private final int begin; private final int end; + private final VID vid; private final String id; /** @@ -44,7 +47,7 @@ public class Unit */ public Unit(int aIndex, int aBegin, int aEnd) { - this(null, aIndex, aBegin, aEnd); + this(null, null, aIndex, aBegin, aEnd); } /** @@ -58,8 +61,9 @@ public Unit(int aIndex, int aBegin, int aEnd) * @param aEnd * end character offset */ - public Unit(@Nullable String aId, int aIndex, int aBegin, int aEnd) + public Unit(@Nullable VID aVid, @Nullable String aId, int aIndex, int aBegin, int aEnd) { + vid = aVid; id = aId; index = aIndex; begin = aBegin; @@ -87,6 +91,11 @@ public String getId() return id; } + public VID getVid() + { + return vid; + } + @Override public String toString() { diff --git a/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java b/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java index 3b0ba0aa434..d75b99aa9cc 100644 --- a/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java +++ b/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java @@ -213,7 +213,8 @@ private void renderComments(GetDocumentResponse aResponse, VDocument aVDoc, } int index = sentenceIndexes.get(fs); - aResponse.addComment(new SentenceComment(index, type, vcomment.getComment())); + aResponse.addComment( + new SentenceComment(VID.of(fs), index, type, vcomment.getComment())); } else { aResponse.addComment( @@ -312,8 +313,8 @@ private void renderBratRowsFromUnits(GetDocumentResponse aResponse, RenderReques // If there is a sentence ID, then make it accessible to the user via a sentence-level // comment. if (isNotBlank(unit.getId())) { - aResponse.addComment(new SentenceComment(unitNum, Comment.ANNOTATOR_NOTES, - String.format("Sentence ID: %s", unit.getId()))); + aResponse.addComment(new SentenceComment(unit.getVid(), unitNum, + Comment.ANNOTATOR_NOTES, String.format("Sentence ID: %s", unit.getId()))); } unitNum++; diff --git a/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/model/SentenceComment.java b/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/model/SentenceComment.java index 91eb148a24a..d15e11b6513 100644 --- a/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/model/SentenceComment.java +++ b/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/model/SentenceComment.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import de.tudarmstadt.ukp.inception.rendering.vmodel.VID; import de.tudarmstadt.ukp.inception.support.json.BeanAsArraySerializer; /** @@ -39,13 +40,23 @@ public SentenceComment() // Nothing to do } - public SentenceComment(int aSentenceIndex, String aCommentType, String aComment) + public SentenceComment(VID aVid, int aSentenceIndex, String aCommentType, String aComment) { - anchor = new Object[] { "sent", aSentenceIndex }; + anchor = new Object[] { "sent", aSentenceIndex, aVid }; commentType = aCommentType; comment = aComment; } + public int getVid() + { + return (int) anchor[0]; + } + + public void getVid(VID aVid) + { + anchor[2] = aVid; + } + public int getSentenceIndex() { return (int) anchor[1]; diff --git a/inception/inception-brat-editor/src/main/ts/src/protocol/Protocol.ts b/inception/inception-brat-editor/src/main/ts/src/protocol/Protocol.ts index 2ecc8b7a965..4d2ec6e4426 100644 --- a/inception/inception-brat-editor/src/main/ts/src/protocol/Protocol.ts +++ b/inception/inception-brat-editor/src/main/ts/src/protocol/Protocol.ts @@ -47,7 +47,7 @@ export type AnnotationCommentDto = [ ] export type SentenceCommentDto = [ - anchor: ['sent', number], + anchor: ['sent', number, VID], commentType: CommentType, comment: string ] diff --git a/inception/inception-brat-editor/src/main/ts/src/visualizer/Comment.ts b/inception/inception-brat-editor/src/main/ts/src/visualizer/Comment.ts index a32ddbf1c99..6326e2cb9f6 100644 --- a/inception/inception-brat-editor/src/main/ts/src/visualizer/Comment.ts +++ b/inception/inception-brat-editor/src/main/ts/src/visualizer/Comment.ts @@ -1,3 +1,5 @@ +import { VID } from "@inception-project/inception-js-api" + /* * ## INCEpTION ## * Licensed to the Technische Universität Darmstadt under one @@ -38,6 +40,7 @@ * SOFTWARE. */ export class Comment { + id: VID = undefined text: string = undefined type: string = undefined diff --git a/inception/inception-brat-editor/src/main/ts/src/visualizer/Visualizer.ts b/inception/inception-brat-editor/src/main/ts/src/visualizer/Visualizer.ts index a2a76e8936b..bba719b0074 100644 --- a/inception/inception-brat-editor/src/main/ts/src/visualizer/Visualizer.ts +++ b/inception/inception-brat-editor/src/main/ts/src/visualizer/Visualizer.ts @@ -735,11 +735,12 @@ export class Visualizer { if (comment[0] instanceof Array && comment[0][0] === 'sent') { // sentence comment const sent = comment[0][1] + const id = comment[0][2] let text = comment[2] if (docData.sentComment[sent]) { text = docData.sentComment[sent].text + '
' + text } - docData.sentComment[sent] = { type: comment[1], text } + docData.sentComment[sent] = { id, type: comment[1], text } continue } @@ -3739,6 +3740,14 @@ export class Visualizer { if (id) { const comment = this.data.sentComment[id] if (comment) { + if (evt.target) { + const fakeSpan = new Span() + fakeSpan.vid = comment.id + fakeSpan.document = { text: this.data.text } + fakeSpan.layer = { id: 0, name: Util.spanDisplayForm(this.entityTypes, comment.type) } + evt.target.dispatchEvent(new AnnotationOverEvent(fakeSpan, evt.originalEvent)) + } + this.dispatcher.post('displaySentComment', [evt, comment.text, comment.type]) } } diff --git a/inception/inception-brat-editor/src/main/ts/src/visualizer_ui/VisualizerUI.ts b/inception/inception-brat-editor/src/main/ts/src/visualizer_ui/VisualizerUI.ts index 78192706f85..c7f9b42e593 100644 --- a/inception/inception-brat-editor/src/main/ts/src/visualizer_ui/VisualizerUI.ts +++ b/inception/inception-brat-editor/src/main/ts/src/visualizer_ui/VisualizerUI.ts @@ -72,7 +72,7 @@ export class VisualizerUI { .on('dataReady', this, this.rememberData) // .on('displaySpanComment', this, this.displaySpanComment) // .on('displayArcComment', this, this.displayArcComment) - .on('displaySentComment', this, this.displaySentComment) +// .on('displaySentComment', this, this.displaySentComment) .on('hideComment', this, this.hideComment) .on('resize', this, this.onResize) .on('spanAndAttributeTypesLoaded', this, this.spanAndAttributeTypesLoaded) diff --git a/inception/inception-diam/pom.xml b/inception/inception-diam/pom.xml index a726f66494e..40fdbaadbbb 100644 --- a/inception/inception-diam/pom.xml +++ b/inception/inception-diam/pom.xml @@ -167,6 +167,10 @@ org.apache.uima uimaj-core + + org.apache.uima + uimafit-core + com.fasterxml.jackson.core @@ -200,11 +204,6 @@ 0.4.0 - - org.apache.uima - uimafit-core - test - org.springframework.security spring-security-crypto diff --git a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/lazydetails/LazyDetailsLookupServiceImpl.java b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/lazydetails/LazyDetailsLookupServiceImpl.java index 9805f4a5036..2bc774f6649 100644 --- a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/lazydetails/LazyDetailsLookupServiceImpl.java +++ b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/lazydetails/LazyDetailsLookupServiceImpl.java @@ -27,18 +27,23 @@ import java.util.List; import org.apache.uima.cas.CAS; +import org.apache.uima.fit.util.FSUtil; import org.apache.wicket.request.IRequestParameters; import org.apache.wicket.util.string.StringValue; +import com.nimbusds.oauth2.sdk.util.StringUtils; + import de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasProvider; import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationFeature; import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; import de.tudarmstadt.ukp.clarin.webanno.model.Project; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.clarin.webanno.security.model.User; +import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.inception.diam.editor.config.DiamAutoConfig; import de.tudarmstadt.ukp.inception.editor.AnnotationEditorExtensionRegistry; import de.tudarmstadt.ukp.inception.rendering.vmodel.VID; +import de.tudarmstadt.ukp.inception.rendering.vmodel.VLazyDetail; import de.tudarmstadt.ukp.inception.rendering.vmodel.VLazyDetailGroup; import de.tudarmstadt.ukp.inception.schema.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.schema.adapter.AnnotationException; @@ -82,22 +87,33 @@ public List lookupLazyDetails(IRequestParameters request, VID a } var cas = aCas.get(); - var layer = findLayer(aVid, cas, layerParam, aDocument.getProject()); var detailGroups = new ArrayList(); + if (isSentence(cas, aVid)) { + var fs = selectFsByAddr(cas, aVid.getId()); + var id = FSUtil.getFeature(fs, Sentence._FeatName_id, String.class); + if (StringUtils.isNotBlank(id)) { + var group = new VLazyDetailGroup(); + group.addDetail(new VLazyDetail(Sentence._FeatName_id, id)); + detailGroups.add(group); + } + } + else { + var layer = findLayer(aVid, cas, layerParam, aDocument.getProject()); - lookupLayerLevelDetails(aVid, cas, windowBeginOffset, windowEndOffset, layer) - .forEach(detailGroups::add); - - for (var feature : annotationService.listAnnotationFeature(layer)) { - lookupExtensionLevelDetails(aVid, aDocument, cas, aUser, feature) + lookupLayerLevelDetails(aVid, cas, windowBeginOffset, windowEndOffset, layer) .forEach(detailGroups::add); - // FIXME: We would like to get feature-level lazy details for the annotation label - // provided by the extension or said otherwise, we want to e.g. get KB details for a - // concept - // feature suggestion... this worked when we used the "query", but now is broken! - lookupFeatureLevelDetail(aVid, cas, feature).forEach(detailGroups::add); + for (var feature : annotationService.listAnnotationFeature(layer)) { + lookupExtensionLevelDetails(aVid, aDocument, cas, aUser, feature) + .forEach(detailGroups::add); + + // FIXME: We would like to get feature-level lazy details for the annotation label + // provided by the extension or said otherwise, we want to e.g. get KB details for a + // concept feature suggestion... this worked when we used the "query", but now is + // broken! + lookupFeatureLevelDetail(aVid, cas, feature).forEach(detailGroups::add); + } } return detailGroups.stream() // @@ -105,6 +121,17 @@ public List lookupLazyDetails(IRequestParameters request, VID a .collect(toList()); } + private boolean isSentence(CAS aCas, VID aVid) + { + if (aVid.isSynthetic()) { + return false; + } + + var fs = selectFsByAddr(aCas, aVid.getId()); + return Sentence._TypeName.equals(fs.getType().getName()); + + } + private LazyDetailGroup toExternalForm(VLazyDetailGroup aGroup) { var extGroup = new LazyDetailGroup(aGroup.getTitle()); diff --git a/inception/inception-js-api/src/main/ts/src/widget/AnnotationDetailPopOver.svelte b/inception/inception-js-api/src/main/ts/src/widget/AnnotationDetailPopOver.svelte index 8f90305df91..f411c021268 100644 --- a/inception/inception-js-api/src/main/ts/src/widget/AnnotationDetailPopOver.svelte +++ b/inception/inception-js-api/src/main/ts/src/widget/AnnotationDetailPopOver.svelte @@ -27,7 +27,7 @@ DiamAjax, LazyDetailGroup, } from "../.."; - import { onMount } from "svelte"; + import { onDestroy, onMount } from "svelte"; export let ajax: DiamAjax; export let root: Element; @@ -44,80 +44,105 @@ let popover: HTMLElement; let detailGroups: LazyDetailGroup[]; let popoverTimeoutId: number | undefined; + let loading = false; onMount(() => { - // React to mouse hovering over annotation - root.addEventListener( - AnnotationOverEvent.eventType, - (e: AnnotationOverEvent) => { - const originalEvent = e.originalEvent; - if (!(originalEvent instanceof MouseEvent)) - return; - - if (popoverTimeoutId) window.clearTimeout(popoverTimeoutId); - popoverTimeoutId = undefined; + root.addEventListener(AnnotationOverEvent.eventType, onAnnotationOver); + root.addEventListener(AnnotationOutEvent.eventType, onAnnotationOut); + root.addEventListener("mousemove", onMouseMove); + root.addEventListener("mousedown", onMouseDown); + }); - // console.log("Popover triggered"); - if (annotation && annotation.vid !== e.annotation.vid) { - annotation = e.annotation; - } else if (!annotation || annotation.vid !== e.annotation.vid) { - popoverTimeoutId = window.setTimeout(() => { - popoverTimeoutId = undefined; - annotation = e.annotation; - popoverTimeoutId = window.setTimeout(() => { - movePopover(originalEvent); - popoverTimeoutId = undefined; - }, renderDelay); - }, showDelay); - } - } - ); - - // Hide popover when leaving the annotation - root.addEventListener( - AnnotationOutEvent.eventType, - (e: AnnotationOutEvent) => { - if (!(e.originalEvent instanceof MouseEvent)) - return; - // console.log("Popover out", e.target) - if (popoverTimeoutId) { - window.clearTimeout(popoverTimeoutId); - popoverTimeoutId = undefined; - } - popoverTimeoutId = window.setTimeout(() => { - if (annotation) { - // console.log("Popover hiding") - annotation = undefined; - } - }, hideDelay); - } - ); + onDestroy(() => { + root.removeEventListener(AnnotationOverEvent.eventType, onAnnotationOver); + root.removeEventListener(AnnotationOutEvent.eventType, onAnnotationOut); + root.removeEventListener("mousemove", onMouseMove); + root.removeEventListener("mousedown", onMouseDown); + }) + + $: { + if (annotation) { + loading = true + ajax.loadLazyDetails(annotation) + .then((response) => { + loading = false + detailGroups = response + }) + .catch(() => { + loading = false + detailGroups = [{ + title: "Error", + details: [{label: "", value: "Unable to load details." + }]}] + }) + } + } + + function onMouseMove(e: MouseEvent) { + if (!annotation) return; + + // if (!popoverTimeoutId && annotation) { + // annotation = undefined + // return + // } + + movePopover(e); + } + + function onMouseDown(e: MouseEvent) { + if (popoverTimeoutId) { + window.clearTimeout(popoverTimeoutId); + popoverTimeoutId = undefined; + } + annotation = undefined; + } + + function onAnnotationOver(e: AnnotationOverEvent) { + const originalEvent = e.originalEvent; + if (!(originalEvent instanceof MouseEvent)) + return; + + if (popoverTimeoutId) window.clearTimeout(popoverTimeoutId); + popoverTimeoutId = undefined; - // Follow the mouse around - root.addEventListener("mousemove", (e: MouseEvent) => { - if (!annotation) return; + if (annotation && annotation.vid !== e.annotation.vid) { + annotation = e.annotation; + detailGroups = undefined + } else if (!annotation || annotation.vid !== e.annotation.vid) { + showPopoverWithDelay(e.annotation, originalEvent) + } + } - movePopover(e); - }); + function onAnnotationOut(e: AnnotationOutEvent) { + if (!(e.originalEvent instanceof MouseEvent)) + return; - root.addEventListener("mousedown", (e: MouseEvent) => { - if (popoverTimeoutId) { - window.clearTimeout(popoverTimeoutId); + hidePopoverWithDelay(); + } + + function showPopoverWithDelay(ann: Annotation, originalEvent: MouseEvent) { + popoverTimeoutId = window.setTimeout(() => { popoverTimeoutId = undefined; - } - annotation = undefined; - }); - }); + annotation = ann; + popoverTimeoutId = window.setTimeout(() => { + movePopover(originalEvent); + popoverTimeoutId = undefined; + }, renderDelay); + }, showDelay); + } - $: { - if (annotation) { - ajax.loadLazyDetails(annotation) - .then((response) => detailGroups = response) - .catch(() => detailGroups = [{ - title: "Error", - details: [{label: "", value: "Unable to load details." - }]}]) + function hidePopoverWithDelay() { + if (popoverTimeoutId) { + window.clearTimeout(popoverTimeoutId); + popoverTimeoutId = undefined; } + + popoverTimeoutId = window.setTimeout(() => { + if (annotation) { + annotation = undefined; + } + popoverTimeoutId = undefined + }, hideDelay); } function movePopover(e: MouseEvent) { @@ -130,7 +155,7 @@ if (y + rect.height + yOffset > window.innerHeight) { top = y - rect.height - yOffset; } else { - top = y + yOffset; + top = y + yOffset; } // Shift left if the popover is about to be clipped on the right @@ -168,7 +193,15 @@
{comment.comment}
{/each} {/if} - {#if detailGroups} + {#if loading} +
+
+ +
+
+ {:else if detailGroups} {#each detailGroups as detailGroup} {#if detailGroup.title}
{detailGroup.title}