From f4a01bc57621cb39962a5907b5ec13e23c8eab51 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 1 Oct 2024 22:08:31 +0200 Subject: [PATCH 001/106] #3415 - Move all configuration options to Spring configuration beans - Move profile accessibility setting to property bean --- .../clarin/webanno/security/UserDaoImpl.java | 14 +++---- .../config/SecurityAutoConfiguration.java | 6 ++- .../config/UserProfileProperties.java | 23 ++++++++++++ .../config/UserProfilePropertiesImpl.java | 37 +++++++++++++++++++ .../ukp/inception/support/SettingsUtil.java | 6 --- 5 files changed, 69 insertions(+), 17 deletions(-) create mode 100644 inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfileProperties.java create mode 100644 inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfilePropertiesImpl.java diff --git a/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/UserDaoImpl.java b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/UserDaoImpl.java index c990a01731c..7d6c9deddb7 100644 --- a/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/UserDaoImpl.java +++ b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/UserDaoImpl.java @@ -39,7 +39,6 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.Properties; import java.util.Set; import java.util.regex.Pattern; @@ -62,11 +61,11 @@ import de.tudarmstadt.ukp.clarin.webanno.security.config.SecurityAutoConfiguration; import de.tudarmstadt.ukp.clarin.webanno.security.config.SecurityProperties; +import de.tudarmstadt.ukp.clarin.webanno.security.config.UserProfileProperties; import de.tudarmstadt.ukp.clarin.webanno.security.model.Authority; import de.tudarmstadt.ukp.clarin.webanno.security.model.Role; import de.tudarmstadt.ukp.clarin.webanno.security.model.User; import de.tudarmstadt.ukp.clarin.webanno.security.model.User_; -import de.tudarmstadt.ukp.inception.support.SettingsUtil; import de.tudarmstadt.ukp.inception.support.spring.ApplicationContextProvider; import de.tudarmstadt.ukp.inception.support.text.TextUtils; import jakarta.persistence.EntityManager; @@ -112,16 +111,19 @@ public class UserDaoImpl private final EntityManager entityManager; private final SecurityProperties securityProperties; + private final UserProfileProperties userProfileProperties; private final PlatformTransactionManager transactionManager; private final SessionRegistry sessionRegistry; public UserDaoImpl(EntityManager aEntityManager, SecurityProperties aSecurityProperties, + UserProfileProperties aUserProfileProperties, PlatformTransactionManager aTransactionManager, SessionRegistry aSessionRegistry) { entityManager = aEntityManager; securityProperties = aSecurityProperties; transactionManager = aTransactionManager; sessionRegistry = aSessionRegistry; + userProfileProperties = aUserProfileProperties; } @EventListener @@ -731,16 +733,10 @@ public boolean canChangePassword(User aUser) return false; // External users and project-bound users cannot } - private static boolean isProfileSelfServiceEnabled() - { - Properties settings = SettingsUtil.getSettings(); - return "true".equals(settings.getProperty(SettingsUtil.CFG_USER_ALLOW_PROFILE_ACCESS)); - } - @Override public boolean isProfileSelfServiceAllowed(User aUser) { - if (!isProfileSelfServiceEnabled()) { + if (!userProfileProperties.isAccessible()) { return false; } diff --git a/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/SecurityAutoConfiguration.java b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/SecurityAutoConfiguration.java index 6be84ba0657..59883038931 100644 --- a/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/SecurityAutoConfiguration.java +++ b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/SecurityAutoConfiguration.java @@ -59,6 +59,7 @@ @Configuration @EnableConfigurationProperties({ // + UserProfilePropertiesImpl.class, // LegacyLoginPropertiesImpl.class, // LoginPropertiesImpl.class, // SecurityPropertiesImpl.class, // @@ -70,10 +71,11 @@ public class SecurityAutoConfiguration @Bean("userRepository") public UserDao userService(SecurityProperties aSecurityProperties, + UserProfileProperties aUserProfileProperties, @Autowired(required = false) SessionRegistry aSessionRegistry) { - return new UserDaoImpl(entityManager, aSecurityProperties, transactionManager, - aSessionRegistry); + return new UserDaoImpl(entityManager, aSecurityProperties, aUserProfileProperties, + transactionManager, aSessionRegistry); } @Bean diff --git a/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfileProperties.java b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfileProperties.java new file mode 100644 index 00000000000..b6199996a23 --- /dev/null +++ b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfileProperties.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.clarin.webanno.security.config; + +public interface UserProfileProperties +{ + boolean isAccessible(); +} diff --git a/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfilePropertiesImpl.java b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfilePropertiesImpl.java new file mode 100644 index 00000000000..9e1bbe063a6 --- /dev/null +++ b/inception/inception-security/src/main/java/de/tudarmstadt/ukp/clarin/webanno/security/config/UserProfilePropertiesImpl.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.clarin.webanno.security.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties("user.profile") +public class UserProfilePropertiesImpl implements UserProfileProperties +{ + private boolean accessible; + + public void setAccessible(boolean aAccessible) + { + accessible = aAccessible; + } + + @Override + public boolean isAccessible() + { + return accessible; + } +} diff --git a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/SettingsUtil.java b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/SettingsUtil.java index d0e0bb5891d..3f53c49e88c 100644 --- a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/SettingsUtil.java +++ b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/SettingsUtil.java @@ -76,12 +76,6 @@ public class SettingsUtil @Deprecated public static final String CFG_WARNINGS_UNSUPPORTED_BROWSER = "warnings.unsupportedBrowser"; - /** - * @deprecated Should introduce/use a Spring properties bean instead. - */ - @Deprecated - public static final String CFG_USER_ALLOW_PROFILE_ACCESS = "user.profile.accessible"; - /** * @deprecated Should introduce/use a Spring properties bean instead. */ From 603e598fe71f07f921ac74e5977cb339e850494a Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 1 Oct 2024 21:07:02 +0200 Subject: [PATCH 002/106] #5078 - Upgrade dependencies - rhino 1.7.14 -> 1.7.15 - ant 1.10.14 -> 1.10.15 - junit 5.11.0 -> 5.11.1 - junit-platform 1.11.0 -> 1.11.1 - mockito 5.12.0 -> 5.14.1 - testcontainers 1.20.1 -> 1.20.2 - spring 6.1.12 -> 6.1.13 - spring-boot 3.3.2 -> 3.3.4 - spring-security 6.3.1 -> 6.3.3 - swagger 2.2.22 -> 2.2.24 - log4j 2.23.1 -> 2.24.1 - tomcat 10.1.28 -> 10.1.30 - jetty 12.0.12 -> 12.0.14 - postgres-driver 42.7.3 -> 43.7.4 - wicket 10.1.0 -> 10.2.0 - wicketstuff 10.1.1 -> 10.2.0 - wicket-bootstrap 7.0.5 -> 7.0.8 - wicket-jquery-selectors 4.0.5 -> 4.0.6 - wicket-webjars 4.0.4 -> 4.0.5 - opensearch 2.16.0 -> 2.17.0 - jackson 2.17.2 -> 2.18.0 - snakeyaml 2.2 -> 2.3 - okio 3.9.0 -> 3.9.1 - byte-buddy 1.14.19 -> 1.15.3 - commons-csv 1.11.0 -> 1.12.0 - commons-lang3 3.16.0 -> 3.17.0 - commons-compress 1.27.0 -> 1.27.1 - commons-io 2.16.1 -> 2.17.0 - jinjava 2.7.2 -> 2.7.3 - nimbus-jose-jwt 9.40 -> 9.41.2 - json-schema-validator 1.5.1 -> 1.5.2 - snappy 1.1.10.6 -> 1.1.10.7 - jquery 1.13.3 -> 1.14.0 --- inception/inception-dependencies/pom.xml | 26 ++++++++- pom.xml | 67 ++++++++++++------------ 2 files changed, 58 insertions(+), 35 deletions(-) diff --git a/inception/inception-dependencies/pom.xml b/inception/inception-dependencies/pom.xml index ba05d0f9cde..47ee07ed650 100644 --- a/inception/inception-dependencies/pom.xml +++ b/inception/inception-dependencies/pom.xml @@ -390,7 +390,7 @@ org.mozilla rhino-runtime - 1.7.14 + 1.7.15 @@ -1372,7 +1372,7 @@ org.apache.ant ant - 1.10.14 + ${ant-version} @@ -1424,6 +1424,28 @@ ${spring.boot.version} pom import + + + org.seleniumhq.selenium + * + + + org.jetbrains.kotlinx + * + + + org.flywaydb + * + + + org.apache.logging.log4j + * + + + org.infinispan + * + + diff --git a/pom.xml b/pom.xml index d94702fbec5..c347ad156ad 100644 --- a/pom.xml +++ b/pom.xml @@ -67,15 +67,15 @@ - 1.10.14 - 4.0.22 + 1.10.15 + 4.0.23 - 5.11.0 - 1.11.0 - 5.12.0 + 5.11.1 + 1.11.1 + 5.14.1 3.26.3 2.10.0 - 1.20.1 + 1.20.2 4.2.2 @@ -86,35 +86,35 @@ 3.0.3 - 6.1.12 - 3.3.2 + 6.1.13 + 3.3.4 3.3.2 - 6.3.1 + 6.3.3 2.6.0 - 2.2.22 + 2.2.24 0.12.6 2.0.16 - 2.23.1 + 2.24.1 3.6.0.Final 7.14.0 - 10.1.28 - 12.0.12 + 10.1.30 + 12.0.14 6.0.0 3.4.1 12.6.3.jre11 9.0.0 - 42.7.3 + 42.7.4 6.5.2.Final 8.0.1.Final - 10.1.0 - 10.1.1 - 7.0.5 - 4.0.5 - 4.0.4 + 10.2.0 + 10.2.0 + 7.0.8 + 4.0.6 + 4.0.5 4.0.0 9.11.1 9.6.1 - 2.16.0 + 2.17.0 5.1.0 5.0.2 5.5.1 @@ -132,39 +132,40 @@ 2.3.1 2.3.18 + 1.10.15 20230227 2.6.0 0.5.0 - 2.17.2 - 2.2 + 2.18.0 + 2.3 4.12.0 - 3.9.0 - 1.14.19 + 3.9.1 + 1.15.3 3.1.8 1.9.4 4.4 - 1.11.0 + 1.12.0 1.5 2.12.0 - 3.16.0 + 3.17.0 2.12.0 1.17.1 - 1.27.0 + 1.27.1 3.6.1 1.12.0 1.9.0 - 2.16.1 + 2.17.0 1.3.3 1.1 - 2.7.2 + 2.7.3 8.5.14 4.7.6 6.7.0 - 9.40 - 1.5.1 + 9.41.2 + 1.5.2 6.10.0.202406032230-r 4.0.5 - 1.1.10.6 + 1.1.10.7 0.13.6 5.14.0 1.18.1 @@ -197,7 +198,7 @@ ^20.0.0 ^3.0.2 3.7.1 - 1.13.3 + 1.14.0 2.14.305 2.11.8 1.8.2 From 7498df77d1974281115168fc657b7c4ba1b86b7e Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 2 Oct 2024 21:22:01 +0200 Subject: [PATCH 003/106] #5082 - Splitting tokens using zones sometimes does not work - Fixed issue and added tests --- .../support/uima/SegmentationUtils.java | 8 ++-- .../support/uima/SegmentationUtilsTest.java | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtils.java b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtils.java index d61f1e4466d..f0a339ab1a9 100644 --- a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtils.java +++ b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtils.java @@ -103,9 +103,11 @@ public static void tokenize(CAS aCas, int aBegin, int aEnd, var innerZoneBoundariesBuffer = new IntArrayList(); innerZoneBoundariesBuffer.add(s.getBegin()); innerZoneBoundariesBuffer.add(s.getEnd()); - while (zbi < sortedZoneBoundaries.length && sortedZoneBoundaries[zbi] >= s.getBegin() - && sortedZoneBoundaries[zbi] < s.getEnd()) { - innerZoneBoundariesBuffer.add(sortedZoneBoundaries[zbi]); + + while (zbi < sortedZoneBoundaries.length && sortedZoneBoundaries[zbi] < s.getEnd()) { + if (sortedZoneBoundaries[zbi] >= s.getBegin()) { + innerZoneBoundariesBuffer.add(sortedZoneBoundaries[zbi]); + } zbi++; } diff --git a/inception/inception-support/src/test/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtilsTest.java b/inception/inception-support/src/test/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtilsTest.java index ba394f568b0..94da9931145 100644 --- a/inception/inception-support/src/test/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtilsTest.java +++ b/inception/inception-support/src/test/java/de/tudarmstadt/ukp/inception/support/uima/SegmentationUtilsTest.java @@ -136,4 +136,46 @@ public void testTokenizeWithLimitAndZones() throws Exception "one", ".", // "i", "a", "m"); } + + @Test + public void testTokenizeWithZonesAndNonDefaultSentence() throws Exception + { + var jcas = createText(" this isa test. ", "en"); + new Sentence(jcas, 4, 18).addToIndexes(); + new Div(jcas, 11, 11).addToIndexes(); + + tokenize(jcas.getCas(), 0, jcas.getDocumentText().length(), jcas.select(Div.class)); + + assertThat(select(jcas, Sentence.class)) // + .extracting(Sentence::getCoveredText) // + .containsExactly("this isa test."); + + assertThat(select(jcas, Token.class)) // + .extracting(Token::getCoveredText) // + .containsExactly( // + "this", "is", "a", "test", "."); + } + + @Test + public void testTokenizeWithZonesInBetweenSentences() throws Exception + { + var jcas = createText("0123456789", "en"); + new Sentence(jcas, 1, 3).addToIndexes(); + new Sentence(jcas, 4, 6).addToIndexes(); + new Sentence(jcas, 7, 9).addToIndexes(); + new Div(jcas, 2, 2).addToIndexes(); + new Div(jcas, 5, 5).addToIndexes(); + new Div(jcas, 8, 8).addToIndexes(); + + tokenize(jcas.getCas(), 0, jcas.getDocumentText().length(), jcas.select(Div.class)); + + assertThat(select(jcas, Sentence.class)) // + .extracting(Sentence::getCoveredText) // + .containsExactly("12", "45", "78"); + + assertThat(select(jcas, Token.class)) // + .extracting(Token::getCoveredText) // + .containsExactly( // + "1", "2", "4", "5", "7", "8"); + } } From 307279821d87321ab53a09f8ee6b8db0d5949342 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 7 Oct 2024 21:24:45 +0200 Subject: [PATCH 004/106] #5085 - Error displaying document when there are no visible layers - Do not exit pre-renderer early when there are no visible layers to avoid leaving the response in an unfinished state - Better handle the case where a document is empty in the brat serializer --- .../api/annotation/rendering/PreRendererImpl.java | 2 +- .../webanno/brat/render/BratSerializerImpl.java | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/rendering/PreRendererImpl.java b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/rendering/PreRendererImpl.java index 35774b75ad0..f58cdfdf2ac 100644 --- a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/rendering/PreRendererImpl.java +++ b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/annotation/rendering/PreRendererImpl.java @@ -98,7 +98,7 @@ public void render(VDocument aResponse, RenderRequest aRequest) Validate.notNull(cas, "CAS cannot be null"); - if (aRequest.getVisibleLayers().isEmpty() || isEmpty(documentText)) { + if (isEmpty(documentText)) { return; } diff --git a/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java b/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java index f11fae1f8ef..5106a241ca5 100644 --- a/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java +++ b/inception/inception-brat-editor/src/main/java/de/tudarmstadt/ukp/clarin/webanno/brat/render/BratSerializerImpl.java @@ -20,6 +20,7 @@ import static de.tudarmstadt.ukp.clarin.webanno.brat.schema.BratSchemaGeneratorImpl.getBratTypeName; import static de.tudarmstadt.ukp.clarin.webanno.model.ScriptDirection.RTL; import static java.util.Arrays.asList; +import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.uima.cas.text.AnnotationPredicates.covering; import static org.apache.uima.cas.text.AnnotationPredicates.overlappingAtBegin; @@ -251,11 +252,11 @@ private static List getArgument(VID aGovernorFs, VID aDependentFs) private void renderText(VDocument aVDoc, GetDocumentResponse aResponse, RenderRequest aRequest) { - if (!aRequest.isIncludeText()) { + if (!aRequest.isIncludeText() || isEmpty(aVDoc.getText())) { return; } - String visibleText = aVDoc.getText(); + var visibleText = aVDoc.getText(); char replacementChar = 0; if (StringUtils.isNotEmpty(properties.getWhiteSpaceReplacementCharacter())) { replacementChar = properties.getWhiteSpaceReplacementCharacter().charAt(0); @@ -267,9 +268,13 @@ private void renderText(VDocument aVDoc, GetDocumentResponse aResponse, RenderRe private void renderBratTokensFromText(GetDocumentResponse aResponse, VDocument aVDoc) { - List bratTokenOffsets = new ArrayList<>(); - String visibleText = aVDoc.getText(); - BreakIterator bi = BreakIterator.getWordInstance(Locale.ROOT); + if (isEmpty(aVDoc.getText())) { + return; + } + + var bratTokenOffsets = new ArrayList(); + var visibleText = aVDoc.getText(); + var bi = BreakIterator.getWordInstance(Locale.ROOT); bi.setText(visibleText); int last = bi.first(); int cur = bi.next(); From bd9581caeeaebf38c9a1334c224ebbb3171e5754 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 8 Oct 2024 12:41:38 +0200 Subject: [PATCH 005/106] #5087 - Interactive recommender sidebar does not invoke the right recommender - Remove the response limit for Ollama - Distingish different auto-activation reasons in the recommender sidebar - Mark interactive recommender as "active" so that the PredictionTask considers them (not yet sure if that is the best approach, but then at least also the "accept best" buttons in the recommender sidebar are there) - Skip interactive recommenders unless they are explicitly included to be run in a PredictionTask - Log a bit more timing information --- .../imls/ollama/OllamaRecommender.java | 7 +- .../api/model/EvaluatedRecommender.java | 12 +++- .../service/RecommendationServiceImpl.java | 6 +- ...NonTrainableRecommenderActivationTask.java | 9 ++- .../recommendation/tasks/PredictionTask.java | 71 +++++++++++++------ .../recommendation/tasks/SelectionTask.java | 31 ++++---- 6 files changed, 88 insertions(+), 48 deletions(-) diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java index dd2264971dc..8770d8f6d27 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java @@ -20,6 +20,7 @@ import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.VAR_EXAMPLES; import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.getPromptContextGenerator; import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ResponseExtractor.getResponseExtractor; +import static java.lang.System.currentTimeMillis; import java.io.IOException; import java.lang.invoke.MethodHandles; @@ -99,11 +100,11 @@ private String query(String aPrompt) throws IOException .withFormat(traits.getFormat()) // .withRaw(traits.isRaw()) // .withStream(false) // - // FIXME: Make NUM_PREDICT accessible in UI - .withOption(OllamaGenerateRequest.NUM_PREDICT, 300) // .build(); + var startTime = currentTimeMillis(); var response = client.generate(traits.getUrl(), request).trim(); - LOG.trace("Ollama [{}] responds: [{}]", traits.getModel(), response); + LOG.trace("Ollama [{}] responds ({} ms): [{}]", traits.getModel(), + currentTimeMillis() - startTime, response); return response; } } diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/EvaluatedRecommender.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/EvaluatedRecommender.java index 575b17e539c..90973e15f53 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/EvaluatedRecommender.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/model/EvaluatedRecommender.java @@ -56,10 +56,16 @@ public String getReasonForState() return reasonForState; } - public static EvaluatedRecommender makeActiveWithoutEvaluation(Recommender aRecommender) + @Override + public String toString() { - return new EvaluatedRecommender(aRecommender, EvaluationResult.skipped(), true, - "Recommender is always active (without evaluation)."); + return "EvaluatedRecommender [" + recommender + " -> " + (active ? "ACTIVE" : "OFF") + "]"; + } + + public static EvaluatedRecommender makeActiveWithoutEvaluation(Recommender aRecommender, + String aReason) + { + return new EvaluatedRecommender(aRecommender, EvaluationResult.skipped(), true, aReason); } public static EvaluatedRecommender makeActive(Recommender aRecommender, diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java index 4270915d8a3..4503ed2f02a 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/service/RecommendationServiceImpl.java @@ -43,7 +43,6 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.collections4.MapIterator; import org.apache.commons.collections4.MultiValuedMap; import org.apache.commons.collections4.multimap.HashSetValuedHashMap; import org.apache.commons.lang3.Validate; @@ -1406,10 +1405,9 @@ public void setPreferences(Preferences aPreferences) public MultiValuedMap getActiveRecommenders() { - MultiValuedMap active = new HashSetValuedHashMap<>(); + var active = new HashSetValuedHashMap(); - MapIterator i = evaluatedRecommenders - .mapIterator(); + var i = evaluatedRecommenders.mapIterator(); while (i.hasNext()) { i.next(); if (i.getValue().isActive()) { diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/NonTrainableRecommenderActivationTask.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/NonTrainableRecommenderActivationTask.java index 4464773aa18..a76bb974187 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/NonTrainableRecommenderActivationTask.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/NonTrainableRecommenderActivationTask.java @@ -146,6 +146,10 @@ private Optional considerRecommender(User user, Recommende var engine = factory.build(recommender); + if (factory.isInteractive(recommender)) { + return Optional.of(activateNonTrainableRecommender(user, recommender, engine)); + } + return switch (engine.getTrainingCapability()) { case TRAINING_NOT_SUPPORTED, TRAINING_SUPPORTED -> Optional .of(activateNonTrainableRecommender(user, recommender, engine)); @@ -169,7 +173,8 @@ private EvaluatedRecommender activateNonTrainableRecommender(User user, Recommen LOG.debug("[{}][{}]: Activating [{}] non-trainable recommender", user.getUsername(), recommenderName, recommenderName); info("Recommender [%s] activated because it is not trainable", recommenderName); - return EvaluatedRecommender.makeActiveWithoutEvaluation(recommender); + return EvaluatedRecommender.makeActiveWithoutEvaluation(recommender, + "Non-trainable recommenders is always active (without evaluation)"); } private EvaluatedRecommender skipTrainableRecommender(User user, Recommender recommender) @@ -237,8 +242,6 @@ public static Builder> builder() public static class Builder> extends RecommendationTask_ImplBase.Builder { - private Recommender recommender; - public NonTrainableRecommenderActivationTask build() { return new NonTrainableRecommenderActivationTask(this); diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java index c301691ed44..30a33769879 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/PredictionTask.java @@ -66,7 +66,6 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommenderContext; import de.tudarmstadt.ukp.inception.recommendation.event.RecommenderTaskNotificationEvent; import de.tudarmstadt.ukp.inception.rendering.model.Range; -import de.tudarmstadt.ukp.inception.scheduling.TaskMonitor; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.StopWatch; import de.tudarmstadt.ukp.inception.support.WebAnnoConst; @@ -196,7 +195,7 @@ private Predictions generatePredictions() } } - return generatePredictionsOnSingleDocument(currentDocument, docs, getMonitor()); + return generatePredictionsOnSingleDocument(currentDocument, docs); } /** @@ -255,7 +254,7 @@ private Predictions generatePredictionsOnAllDocuments(List aDocu * @return the new predictions. */ private Predictions generatePredictionsOnSingleDocument(SourceDocument aCurrentDocument, - List aDocuments, TaskMonitor aMonitor) + List aDocuments) { var sessionOwner = getSessionOwner(); var project = getProject(); @@ -264,7 +263,7 @@ private Predictions generatePredictionsOnSingleDocument(SourceDocument aCurrentD ? new Predictions(predecessorPredictions) : new Predictions(sessionOwner, dataOwner, project); - aMonitor.setMaxProgress(1); + getMonitor().setMaxProgress(1); if (predecessorPredictions != null) { // Limit prediction to a single document and inherit the rest @@ -308,7 +307,7 @@ private Predictions generatePredictionsOnSingleDocument(SourceDocument aCurrentD logErrorCreationPredictionCas(incomingPredictions); } - aMonitor.setProgress(1); + getMonitor().setProgress(1); return incomingPredictions; } @@ -354,15 +353,22 @@ private void applyActiveRecommendersToDocument(Predictions aActivePredictions, try { var originalCas = new LazyCas(aDocument); for (var activeRecommender : activeRecommenders) { - // Make sure we have the latest recommender config from the DB - the one - // from the active recommenders list may be outdated var rec = activeRecommender.getRecommender(); - try { - rec = recommendationService.getRecommender(rec.getId()); + // If a recommender is explicitly requested, the configuration from the requested + // recommender object takes precedence over what is stored in the database + if (!recommenders.isEmpty() && recommenders.contains(rec)) { + rec = recommenders.get(recommenders.indexOf(rec)); } - catch (NoResultException e) { - logRecommenderNotAvailable(aPredictions, rec); - continue; + else { + // Make sure we have the latest recommender config from the DB - the one + // from the active recommenders list may be outdated + try { + rec = recommendationService.getRecommender(rec.getId()); + } + catch (NoResultException e) { + logRecommenderNotAvailable(aPredictions, rec); + continue; + } } applySingleRecomenderToDocument(originalCas, rec, aActivePredictions, aPredictions, @@ -425,6 +431,17 @@ private void applySingleRecomenderToDocument(LazyCas aOriginalCas, Recommender a return; } + if (recommenders.isEmpty() && factory.isInteractive(aRecommender)) { + logSkippingInteractiveRecommenderNotExplicitlyRequested(aPredictions, aRecommender); + + if (activePredictions != null) { + inheritSuggestionsAtRecommenderLevel(aPredictions, aRecommender, activePredictions, + aDocument); + } + + return; + } + var engine = factory.build(aRecommender); var isSynchronous = factory.isSynchronous(aRecommender); @@ -467,9 +484,11 @@ private void applySingleRecomenderToDocument(LazyCas aOriginalCas, Recommender a return; } - // If the recommender is not trainable and not sensitive to annotations, we can actually + // If the recommender is not trainable and not sensitive to user input/annotations, we can + // actually // re-use the predictions. - if (TRAINING_NOT_SUPPORTED == engine.getTrainingCapability() + if (!factory.isInteractive(aRecommender) + && TRAINING_NOT_SUPPORTED == engine.getTrainingCapability() && PREDICTION_USES_TEXT_ONLY == engine.getPredictionCapability() && activePredictions != null && activePredictions.hasRunPredictionOnDocument(aDocument)) { @@ -536,6 +555,7 @@ private void invokeRecommender(Predictions aIncomingPredictions, PredictionConte var suggestionSupport = maybeSuggestionSupport.get(); // Perform the actual prediction + var startTime = System.currentTimeMillis(); var predictedRange = predict(aIncomingPredictions, aCtx, aEngine, aPredictionCas, aPredictionRange); @@ -547,7 +567,7 @@ private void invokeRecommender(Predictions aIncomingPredictions, PredictionConte generatedSuggestions); logGeneratedPredictions(aIncomingPredictions, aDocument, rec, predictedRange, - generatedSuggestions, reconciliationResult); + generatedSuggestions, reconciliationResult, currentTimeMillis() - startTime); // Inherit suggestions that are outside the range which was predicted. Note that the engine // might actually predict a different range from what was requested. If the prediction @@ -837,17 +857,17 @@ private void logPredictionStartedForAllDocuments(List docs) private void logGeneratedPredictions(Predictions aIncomingPredictions, SourceDocument aDocument, Recommender aRecommender, Range predictedRange, List generatedSuggestions, - ReconciliationResult reconciliationResult) + ReconciliationResult reconciliationResult, long aDuration) { LOG.debug( - "{} for user {} on document {} in project {} generated {} predictions within range {} (+{}/-{}/={})", + "{} for user {} on document {} in project {} generated {} predictions within range {} (+{}/-{}/={}) ({} ms)", aRecommender, getSessionOwner(), aDocument, aRecommender.getProject(), generatedSuggestions.size(), predictedRange, reconciliationResult.added, - reconciliationResult.removed, reconciliationResult.aged); + reconciliationResult.removed, reconciliationResult.aged, aDuration); aIncomingPredictions.log(LogMessage.info(aRecommender.getName(), // - "Generated [%d] predictions within range %s (+%d/-%d/=%d)", + "Generated [%d] predictions within range %s (+%d/-%d/=%d) (%d ms)", generatedSuggestions.size(), predictedRange, reconciliationResult.added, - reconciliationResult.removed, reconciliationResult.aged)); + reconciliationResult.removed, reconciliationResult.aged, aDuration)); } private void logRecommenderContextNoReady(Predictions aPredictions, SourceDocument aDocument, @@ -869,6 +889,17 @@ private void logSkippingNotRequestedRecommender(Predictions aPredictions, getSessionOwner().getUsername(), aRecommender.getName()); } + private void logSkippingInteractiveRecommenderNotExplicitlyRequested(Predictions aPredictions, + Recommender aRecommender) + { + aPredictions.log(LogMessage.info(aRecommender.getName(), + "Interactive recommender not requested for this run... skipping")); + LOG.info( + "[{}][{}]: Interactive recommender not requested for this run " + + "- skipping recommender", + getSessionOwner().getUsername(), aRecommender.getName()); + } + private void logSkippingSynchronous(Predictions aPredictions, Recommender aRecommender) { aPredictions.log(LogMessage.info(aRecommender.getName(), diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java index 73bc6f440a4..909978f7f88 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/tasks/SelectionTask.java @@ -148,7 +148,7 @@ protected List initialize() seenRecommender = true; } - Recommender recommender = optRecommender.get(); + var recommender = optRecommender.get(); try { long start = System.currentTimeMillis(); @@ -281,7 +281,7 @@ private Optional evaluate(User user, Recommender recommend } if (factory.isInteractive(recommender)) { - return Optional.of(skipInteractiveRecommender(user, recommender)); + return Optional.of(activateInteractiveRecommender(user, recommender)); } if (recommender.isAlwaysSelected()) { @@ -359,31 +359,32 @@ private EvaluatedRecommender activateNonEvaluatableRecommender(String userName, Recommender recommender) { String recommenderName = recommender.getName(); - LOG.debug("[{}][{}]: Activating [{}] without evaluating - not evaluable", userName, + LOG.debug("[{}][{}]: Activating [{}] without evaluation - not evaluable", userName, recommenderName, recommenderName); - info("Recommender [%s] activated without evaluating - not evaluable", recommenderName); - return EvaluatedRecommender.makeActiveWithoutEvaluation(recommender); + info("Recommender [%s] activated without evaluation - not evaluable", recommenderName); + return EvaluatedRecommender.makeActiveWithoutEvaluation(recommender, + "Non-evaluatable recommender is always active (without evaluation)."); } private EvaluatedRecommender activateAlwaysOnRecommender(String userName, Recommender recommender) { String recommenderName = recommender.getName(); - LOG.debug("[{}][{}]: Activating [{}] without evaluating - always selected", userName, + LOG.debug("[{}][{}]: Activating [{}] without evaluation - always selected", userName, recommenderName, recommenderName); - info("Recommender [%s] activated without evaluating - always selected", recommenderName); - return EvaluatedRecommender.makeActiveWithoutEvaluation(recommender); + info("Recommender [%s] activated without evaluation - always selected", recommenderName); + return EvaluatedRecommender.makeActiveWithoutEvaluation(recommender, + "Recommender is always active (without evaluation)."); } - private EvaluatedRecommender skipInteractiveRecommender(User user, Recommender recommender) + private EvaluatedRecommender activateInteractiveRecommender(User user, Recommender recommender) { var recommenderName = recommender.getName(); - LOG.info("[{}][{}]: Recommender reserved for interactive use " + "- skipping recommender", - user.getUsername(), recommenderName); - info("Recommender [%s] reserved for interactive use - skipping recommender", - recommenderName); - return EvaluatedRecommender.makeInactiveWithoutEvaluation(recommender, - "Reserved for interactive use"); + LOG.info("[{}][{}]: Activating [{}] without evaluation - interactive use", + user.getUsername(), recommenderName, recommenderName); + info("Recommender [%s] without evaluation - interactive use", recommenderName); + return EvaluatedRecommender.makeActiveWithoutEvaluation(recommender, + "Interactive recommender is always active (without evaluation)."); } private EvaluatedRecommender skipRecommenderWithInvalidSettings(User user, From ec55e2b8f1d93b72a2ca4dc75f9eaa2eca01d9c9 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 13 Oct 2024 21:14:09 +0200 Subject: [PATCH 006/106] #5089 - Document structure is not retained when preparing a document as curation target - Transfer document structure when preparing the curation CAS --- .../inception/curation/merge/CasMerge.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java index 63fb7e10a4b..6b7132f51bd 100644 --- a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java +++ b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java @@ -41,6 +41,10 @@ import org.apache.uima.cas.CAS; import org.apache.uima.cas.FeatureStructure; import org.apache.uima.cas.text.AnnotationFS; +import org.apache.uima.util.CasCopier; +import org.dkpro.core.api.xml.type.XmlAttribute; +import org.dkpro.core.api.xml.type.XmlDocument; +import org.dkpro.core.api.xml.type.XmlNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; @@ -412,25 +416,40 @@ private void clearAnnotations(SourceDocument aDocument, CAS aCas) throws UIMAExc aCas.setDocumentText(backup.getDocumentText()); transferSegmentation(aDocument.getProject(), aCas, backup); + transferDocumentStructure(aDocument.getProject(), aCas, backup); + } + + private void transferDocumentStructure(Project aProject, CAS aTarget, CAS aSource) + { + var casCopier = new CasCopier(aSource, aTarget); + // Recursively copy the structure - this does not add the copied annotations to the index + for (var doc : aSource.select(XmlDocument.class)) { + casCopier.copyFs(doc); + } + + // Add the document structure annotations to the index + aTarget.select(XmlDocument.class).forEach(aTarget::addFsToIndexes); + aTarget.select(XmlNode.class).forEach(aTarget::addFsToIndexes); + aTarget.select(XmlAttribute.class).forEach(aTarget::addFsToIndexes); } /** * If tokens and/or sentences are not editable, then they are not part of the curation process * and we transfer them from the template CAS. */ - private void transferSegmentation(Project aProject, CAS aCas, CAS backup) + private void transferSegmentation(Project aProject, CAS aTarget, CAS aSource) { if (!schemaService.isTokenLayerEditable(aProject)) { // Transfer token boundaries - for (var t : selectTokens(backup)) { - aCas.addFsToIndexes(createToken(aCas, t.getBegin(), t.getEnd())); + for (var t : selectTokens(aSource)) { + aTarget.addFsToIndexes(createToken(aTarget, t.getBegin(), t.getEnd())); } } if (!schemaService.isSentenceLayerEditable(aProject)) { // Transfer sentence boundaries - for (var s : selectSentences(backup)) { - aCas.addFsToIndexes(createSentence(aCas, s.getBegin(), s.getEnd())); + for (var s : selectSentences(aSource)) { + aTarget.addFsToIndexes(createSentence(aTarget, s.getBegin(), s.getEnd())); } } } From 7668a097a21a186cfc7623738935049ad56db0b0 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 14 Oct 2024 20:36:57 +0200 Subject: [PATCH 007/106] #5089 - Document structure is not retained when preparing a document as curation target - Transfer document structure when preparing the curation CAS - Offer check for missing document structure in curation CAS - Offer repair for missing document structure in curation CAS --- .../casstorage/CasStorageServiceAction.java | 4 +- .../storage/CasStorageServiceImpl.java | 116 +++++++----------- inception/inception-curation/pom.xml | 4 + .../inception/curation/merge/CasMerge.java | 21 +--- inception/inception-diag/pom.xml | 8 ++ .../ukp/clarin/webanno/diag/CasDoctor.java | 59 ++++----- ...tationsStartAndEndWithCharactersCheck.java | 7 +- ...ationsStartAndEndWithinSentencesCheck.java | 13 +- .../AllFeatureStructuresIndexedCheck.java | 5 +- .../checks/CASMetadataTypeIsPresentCheck.java | 5 +- .../ukp/clarin/webanno/diag/checks/Check.java | 5 +- .../diag/checks/DanglingRelationsCheck.java | 25 ++-- .../DocumentTextStartsWithBomCheck.java | 5 +- ...chedSpanAnnotationsTrulyAttachedCheck.java | 11 +- .../LinksReachableThroughChainsCheck.java | 13 +- .../checks/NegativeSizeAnnotationsCheck.java | 5 +- .../NoMultipleIncomingRelationsCheck.java | 20 ++- .../NoZeroSizeTokensAndSentencesCheck.java | 5 +- .../diag/checks/RelationOffsetsCheck.java | 13 +- ...okensAndSententencedDoNotOverlapCheck.java | 5 +- .../checks/UniqueDocumentAnnotationCheck.java | 5 +- .../checks/UnreachableAnnotationsCheck.java | 5 +- ...XmlStructurePresentInCurationCasCheck.java | 78 ++++++++++++ .../config/CasDoctorAutoConfiguration.java | 17 +++ .../CoverAllTextInSentencesRepair.java | 5 +- ...dSpanAnnotationsAndDeleteExtrasRepair.java | 35 +++--- ...hFeatureAttachedSpanAnnotationsRepair.java | 23 ++-- ...xFeatureAttachedSpanAnnotationsRepair.java | 15 ++- .../diag/repairs/RelationOffsetsRepair.java | 15 ++- .../webanno/diag/repairs/RemoveBomRepair.java | 5 +- .../RemoveDanglingChainLinksRepair.java | 7 +- ...gFeatureAttachedSpanAnnotationsRepair.java | 7 +- .../RemoveDanglingRelationsRepair.java | 7 +- ...emoveZeroSizeTokensAndSentencesRepair.java | 5 +- .../clarin/webanno/diag/repairs/Repair.java | 4 +- ...eplaceXmlStructureInCurationCasRepair.java | 81 ++++++++++++ ...ndEndOnNegativeSizedAnnotationsRepair.java | 5 +- .../diag/repairs/TrimAnnotationsRepair.java | 7 +- .../diag/repairs/UpgradeCasRepair.java | 7 +- .../asciidoc/user-guide/casdoctor.adoc | 48 +++++--- .../AllAnnotationsIndexedCheckTest.java | 4 +- ...onsStartAndEndWithCharactersCheckTest.java | 10 +- ...nsStartAndEndWithinSentencesCheckTest.java | 12 +- .../NegativeSizeAnnotationsCheckTest.java | 9 +- .../NoMultipleIncomingRelationsCheckTest.java | 98 ++++++++------- ...sAndSententencedDoNotOverlapCheckTest.java | 11 +- .../CoverAllTextInSentencesRepairTest.java | 15 +-- .../diag/repairs/RemoveBomRepairTest.java | 2 +- .../RemoveDanglingRelationsRepairTest.java | 20 ++- ...dOnNegativeSizedAnnotationsRepairTest.java | 5 +- .../DocumentImportExportServiceImpl.java | 9 +- .../io/xml/dkprocore/XmlNodeUtils.java | 39 +++++- .../ui/project/casdoctor/CheckTask.java | 9 +- .../ui/project/casdoctor/RepairTask.java | 9 +- 54 files changed, 613 insertions(+), 369 deletions(-) create mode 100644 inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/XmlStructurePresentInCurationCasCheck.java create mode 100644 inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReplaceXmlStructureInCurationCasRepair.java diff --git a/inception/inception-annotation-storage-api/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/casstorage/CasStorageServiceAction.java b/inception/inception-annotation-storage-api/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/casstorage/CasStorageServiceAction.java index 4bdeb97f06a..a6131b491f0 100644 --- a/inception/inception-annotation-storage-api/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/casstorage/CasStorageServiceAction.java +++ b/inception/inception-annotation-storage-api/src/main/java/de/tudarmstadt/ukp/clarin/webanno/api/casstorage/CasStorageServiceAction.java @@ -19,8 +19,10 @@ import org.apache.uima.cas.CAS; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; + @FunctionalInterface public interface CasStorageServiceAction { - void apply(CAS aCas) throws Exception; + void apply(SourceDocument aDocument, String aDataOwner, CAS aCas) throws Exception; } diff --git a/inception/inception-annotation-storage/src/main/java/de/tudarmstadt/ukp/inception/annotation/storage/CasStorageServiceImpl.java b/inception/inception-annotation-storage/src/main/java/de/tudarmstadt/ukp/inception/annotation/storage/CasStorageServiceImpl.java index 91abb9b388c..da6d93314a2 100644 --- a/inception/inception-annotation-storage/src/main/java/de/tudarmstadt/ukp/inception/annotation/storage/CasStorageServiceImpl.java +++ b/inception/inception-annotation-storage/src/main/java/de/tudarmstadt/ukp/inception/annotation/storage/CasStorageServiceImpl.java @@ -76,7 +76,6 @@ import de.tudarmstadt.ukp.clarin.webanno.api.casstorage.ConcurentCasModificationException; import de.tudarmstadt.ukp.clarin.webanno.diag.CasDoctor; import de.tudarmstadt.ukp.clarin.webanno.diag.CasDoctorException; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.storage.config.CasStorageCacheProperties; import de.tudarmstadt.ukp.inception.annotation.storage.config.CasStorageServiceAutoConfiguration; @@ -679,34 +678,11 @@ public boolean deleteCas(SourceDocument aDocument, String aUsername) } @Override - public void analyzeAndRepair(SourceDocument aDocument, String aUsername, CAS aCas) + public void analyzeAndRepair(SourceDocument aDocument, String aDataOwner, CAS aCas) { - analyzeAndRepair(aDocument.getProject(), aDocument.getName(), aDocument.getId(), aUsername, - aCas); - } + var project = aDocument.getProject(); - /** - * Runs {@link CasDoctor} in repair mode on the given CAS (if repairs are active), otherwise it - * runs only in analysis mode. - *

- * Note: {@link CasDoctor} is an optional service. If no {@link CasDoctor} implementation - * is available, this method returns without doing anything. - * - * @param aProject - * the project - * @param aDocumentName - * the document name (used for logging) - * @param aDocumentId - * the aDocument ID (used for logging) - * @param aUsername - * the user owning the CAS (used for logging) - * @param aCas - * the CAS object - */ - private void analyzeAndRepair(Project aProject, String aDocumentName, long aDocumentId, - String aUsername, CAS aCas) - { - try (var logCtx = withProjectLogger(aProject)) { + try (var logCtx = withProjectLogger(project)) { if (casDoctor == null) { return; } @@ -715,18 +691,18 @@ private void analyzeAndRepair(Project aProject, String aDocumentName, long aDocu // because the repairs do an analysis as a pre- and post-condition. if (casDoctor.isRepairsActive()) { try { - casDoctor.repair(aProject, aCas); + casDoctor.repair(aDocument, aDataOwner, aCas); } catch (Exception e) { - throw new DataRetrievalFailureException("Error repairing CAS of user [" - + aUsername + "] for document [" + aDocumentName + "] (" + aDocumentId - + ") in project[" + aProject.getName() + "] (" + aProject.getId() + ")", + throw new DataRetrievalFailureException( + "Error repairing CAS of user [" + aDataOwner + "] for document " + + aDocument + " in project " + aDocument.getProject(), e); } } // If the repairs are not active, then we run the analysis explicitly else { - analyze(aProject, aDocumentName, aDocumentId, aUsername, aCas); + analyze(aDocument, aDataOwner, aCas); } } } @@ -737,59 +713,55 @@ private void analyzeAndRepair(Project aProject, String aDocumentName, long aDocu * Note: {@link CasDoctor} is an optional service. If no {@link CasDoctor} implementation * is available, this method returns without doing anything. * - * @param aProject - * the project - * @param aDocumentName - * the document name (used for logging) - * @param aDocumentId - * the aDocument ID (used for logging) - * @param aUsername + * @param aDocument + * the document + * @param aDataOwner * the user owning the CAS (used for logging) * @param aCas * the CAS object */ - private void analyze(Project aProject, String aDocumentName, long aDocumentId, String aUsername, - CAS aCas) + private void analyze(SourceDocument aDocument, String aDataOwner, CAS aCas) { if (casDoctor == null) { return; } + var project = aDocument.getProject(); + try { - casDoctor.analyze(aProject, aCas); + casDoctor.analyze(aDocument, aDataOwner, aCas); } catch (CasDoctorException e) { var detailMsg = new StringBuilder(); - detailMsg.append("CAS Doctor found problems for user [").append(aUsername) - .append("] in document [").append(aDocumentName).append("] (") - .append(aDocumentId).append(") in project [").append(aProject.getName()) - .append("] (").append(aProject.getId()).append(")\n"); + detailMsg.append("CAS Doctor found problems for user [").append(aDataOwner) + .append("] in document ").append(aDocument).append(" in project ") + .append(project).append("\n"); e.getDetails().forEach( m -> detailMsg.append(String.format("- [%s] %s%n", m.level, m.message))); throw new DataRetrievalFailureException(detailMsg.toString()); } catch (Exception e) { - throw new DataRetrievalFailureException("Error analyzing CAS of user [" + aUsername - + "] in document [" + aDocumentName + "] (" + aDocumentId + ") in project[" - + aProject.getName() + "] (" + aProject.getId() + ")", e); + throw new DataRetrievalFailureException("Error analyzing CAS of user [" + aDataOwner + + "] in document " + aDocument + " in project " + project, e); } } @Override - public void exportCas(SourceDocument aDocument, String aUser, OutputStream aStream) + public void exportCas(SourceDocument aDocument, String aDataOwner, OutputStream aStream) throws IOException { // Ensure that the CAS is not being re-written and temporarily unavailable while we export // it, then add this info to a mini-session to ensure that write-access is known try (var session = CasStorageSession.openNested(true)) { - try (var access = new WithExclusiveAccess(aDocument, aUser)) { - session.add(aDocument.getId(), aUser, EXCLUSIVE_WRITE_ACCESS, access.getHolder()); + try (var access = new WithExclusiveAccess(aDocument, aDataOwner)) { + session.add(aDocument.getId(), aDataOwner, EXCLUSIVE_WRITE_ACCESS, + access.getHolder()); - driver.exportCas(aDocument, aUser, aStream); + driver.exportCas(aDocument, aDataOwner, aStream); } finally { - session.remove(aDocument.getId(), aUser); + session.remove(aDocument.getId(), aDataOwner); } } catch (IOException e) { @@ -801,19 +773,20 @@ public void exportCas(SourceDocument aDocument, String aUser, OutputStream aStre } @Override - public void importCas(SourceDocument aDocument, String aUser, InputStream aStream) + public void importCas(SourceDocument aDocument, String aDataOwner, InputStream aStream) throws IOException { // Ensure that the CAS is not being re-written and temporarily unavailable while we export // it, then add this info to a mini-session to ensure that write-access is known try (var session = CasStorageSession.openNested(true)) { - try (var access = new WithExclusiveAccess(aDocument, aUser)) { - session.add(aDocument.getId(), aUser, EXCLUSIVE_WRITE_ACCESS, access.getHolder()); + try (var access = new WithExclusiveAccess(aDocument, aDataOwner)) { + session.add(aDocument.getId(), aDataOwner, EXCLUSIVE_WRITE_ACCESS, + access.getHolder()); - driver.importCas(aDocument, aUser, aStream); + driver.importCas(aDocument, aDataOwner, aStream); } finally { - session.remove(aDocument.getId(), aUser); + session.remove(aDocument.getId(), aDataOwner); } } catch (IOException e) { @@ -825,39 +798,40 @@ public void importCas(SourceDocument aDocument, String aUser, InputStream aStrea } @Override - public void upgradeCas(SourceDocument aDocument, String aUser) throws IOException + public void upgradeCas(SourceDocument aDocument, String aDataOwner) throws IOException { Validate.notNull(aDocument, "Source document must be specified"); - Validate.notBlank(aUser, "User must be specified"); + Validate.notBlank(aDataOwner, "Data owner must be specified"); - forceActionOnCas(aDocument, aUser, // + forceActionOnCas(aDocument, aDataOwner, // (doc, user) -> driver.readCas(doc, user), - (cas) -> schemaService.upgradeCas(cas, aDocument, aUser), // + (doc, user, cas) -> schemaService.upgradeCas(cas, doc, user), // true); } @Override - public void forceActionOnCas(SourceDocument aDocument, String aUser, + public void forceActionOnCas(SourceDocument aDocument, String aDataOwner, CasStorageServiceLoader aLoader, CasStorageServiceAction aAction, boolean aSave) throws IOException { // Ensure that the CAS is not being re-written and temporarily unavailable while we check // upgrade it, then add this info to a mini-session to ensure that write-access is known try (var session = CasStorageSession.openNested(true)) { - try (var access = new WithExclusiveAccess(aDocument, aUser)) { - session.add(aDocument.getId(), aUser, EXCLUSIVE_WRITE_ACCESS, access.getHolder()); + try (var access = new WithExclusiveAccess(aDocument, aDataOwner)) { + session.add(aDocument.getId(), aDataOwner, EXCLUSIVE_WRITE_ACCESS, + access.getHolder()); - var cas = aLoader.load(aDocument, aUser); + var cas = aLoader.load(aDocument, aDataOwner); access.setCas(cas); - aAction.apply(cas); + aAction.apply(aDocument, aDataOwner, cas); if (aSave) { - realWriteCas(aDocument, aUser, cas); + realWriteCas(aDocument, aDataOwner, cas); } } finally { - session.remove(aDocument.getId(), aUser); + session.remove(aDocument.getId(), aDataOwner); } } catch (IOException e) { @@ -1118,7 +1092,7 @@ public void beforeLayerConfigurationChanged(LayerConfigurationChangedEvent aEven private void realWriteCas(SourceDocument aDocument, String aUserName, CAS aCas) throws IOException { - analyze(aDocument.getProject(), aDocument.getName(), aDocument.getId(), aUserName, aCas); + analyze(aDocument, aUserName, aCas); if (CasStorageSession.exists()) { var session = CasStorageSession.get(); diff --git a/inception/inception-curation/pom.xml b/inception/inception-curation/pom.xml index 279f61d3951..297062c480a 100644 --- a/inception/inception-curation/pom.xml +++ b/inception/inception-curation/pom.xml @@ -53,6 +53,10 @@ de.tudarmstadt.ukp.inception.app inception-schema-api + + de.tudarmstadt.ukp.inception.app + inception-io-xml + de.tudarmstadt.ukp.inception.app inception-model diff --git a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java index 6b7132f51bd..4862f810d6c 100644 --- a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java +++ b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/CasMerge.java @@ -41,10 +41,6 @@ import org.apache.uima.cas.CAS; import org.apache.uima.cas.FeatureStructure; import org.apache.uima.cas.text.AnnotationFS; -import org.apache.uima.util.CasCopier; -import org.dkpro.core.api.xml.type.XmlAttribute; -import org.dkpro.core.api.xml.type.XmlDocument; -import org.dkpro.core.api.xml.type.XmlNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; @@ -66,6 +62,7 @@ import de.tudarmstadt.ukp.inception.annotation.storage.CasMetadataUtils; import de.tudarmstadt.ukp.inception.curation.merge.strategy.DefaultMergeStrategy; import de.tudarmstadt.ukp.inception.curation.merge.strategy.MergeStrategy; +import de.tudarmstadt.ukp.inception.io.xml.dkprocore.XmlNodeUtils; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.schema.api.adapter.AnnotationException; import de.tudarmstadt.ukp.inception.schema.api.adapter.IllegalFeatureValueException; @@ -416,21 +413,7 @@ private void clearAnnotations(SourceDocument aDocument, CAS aCas) throws UIMAExc aCas.setDocumentText(backup.getDocumentText()); transferSegmentation(aDocument.getProject(), aCas, backup); - transferDocumentStructure(aDocument.getProject(), aCas, backup); - } - - private void transferDocumentStructure(Project aProject, CAS aTarget, CAS aSource) - { - var casCopier = new CasCopier(aSource, aTarget); - // Recursively copy the structure - this does not add the copied annotations to the index - for (var doc : aSource.select(XmlDocument.class)) { - casCopier.copyFs(doc); - } - - // Add the document structure annotations to the index - aTarget.select(XmlDocument.class).forEach(aTarget::addFsToIndexes); - aTarget.select(XmlNode.class).forEach(aTarget::addFsToIndexes); - aTarget.select(XmlAttribute.class).forEach(aTarget::addFsToIndexes); + XmlNodeUtils.transferXmlDocumentStructure(aCas, backup); } /** diff --git a/inception/inception-diag/pom.xml b/inception/inception-diag/pom.xml index 146c07e774a..0095d5b575a 100644 --- a/inception/inception-diag/pom.xml +++ b/inception/inception-diag/pom.xml @@ -56,6 +56,10 @@ de.tudarmstadt.ukp.inception.app inception-annotation-storage-api + + de.tudarmstadt.ukp.inception.app + inception-documents-api + de.tudarmstadt.ukp.inception.app inception-model @@ -64,6 +68,10 @@ de.tudarmstadt.ukp.inception.app inception-support + + de.tudarmstadt.ukp.inception.app + inception-io-xml + de.tudarmstadt.ukp.inception.app inception-api-annotation diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/CasDoctor.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/CasDoctor.java index 052fe103d66..0f9cafd6a8c 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/CasDoctor.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/CasDoctor.java @@ -34,10 +34,8 @@ import org.springframework.boot.context.event.ApplicationStartedEvent; import org.springframework.context.event.EventListener; -import de.tudarmstadt.ukp.clarin.webanno.diag.checks.Check; import de.tudarmstadt.ukp.clarin.webanno.diag.config.CasDoctorProperties; -import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.SettingsUtil; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -101,10 +99,10 @@ public boolean isFatalChecks() return fatalChecks; } - public void repair(Project aProject, CAS aCas) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas) { - List messages = new ArrayList<>(); - repair(aProject, aCas, messages); + var messages = new ArrayList(); + repair(aDocument, aDataOwner, aCas, messages); if (LOG.isWarnEnabled() && !messages.isEmpty()) { messages.forEach(s -> LOG.warn("{}", s)); } @@ -115,17 +113,18 @@ public boolean isRepairsActive() return !activeRepairs.isEmpty(); } - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { // APPLY REPAIRS - long tStart = currentTimeMillis(); + var tStart = currentTimeMillis(); for (String repairId : activeRepairs) { try { - Repair repair = repairsRegistry.getExtension(repairId).orElseThrow( + var repair = repairsRegistry.getExtension(repairId).orElseThrow( () -> new NoSuchElementException("Unknown repair [" + repairId + "]")); - long tStartTask = currentTimeMillis(); + var tStartTask = currentTimeMillis(); LOG.info("CasDoctor repair [" + repair.getId() + "] running..."); - repair.repair(aProject, aCas, aMessages); + repair.repair(aDocument, aDataOwner, aCas, aMessages); LOG.info("CasDoctor repair [" + repair.getId() + "] completed in " + (currentTimeMillis() - tStartTask) + "ms"); } @@ -140,44 +139,46 @@ public void repair(Project aProject, CAS aCas, List aMessages) // POST-CONDITION: CAS must be consistent // Ensure that the repairs actually fixed the CAS - analyze(aProject, aCas, aMessages, true); + analyze(aDocument, aDataOwner, aCas, aMessages, true); } - public boolean analyze(Project aProject, CAS aCas) throws CasDoctorException + public boolean analyze(SourceDocument aDocument, String aDataOwner, CAS aCas) + throws CasDoctorException { - List messages = new ArrayList<>(); - boolean result = analyze(aProject, aCas, messages); + var messages = new ArrayList(); + var result = analyze(aDocument, aDataOwner, aCas, messages); if (LOG.isDebugEnabled()) { messages.forEach(s -> LOG.debug("{}", s)); } return result; } - public boolean analyze(Project aProject, CAS aCas, List aMessages) + public boolean analyze(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) throws CasDoctorException { - return analyze(aProject, aCas, aMessages, isFatalChecks()); + return analyze(aDocument, aDataOwner, aCas, aMessages, isFatalChecks()); } - public boolean analyze(Project aProject, CAS aCas, List aMessages, - boolean aFatalChecks) + public boolean analyze(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages, boolean aFatalChecks) throws CasDoctorException { if (activeChecks.isEmpty()) { return true; } - long tStart = currentTimeMillis(); + var tStart = currentTimeMillis(); - boolean ok = true; - for (String checkId : activeChecks) { + var ok = true; + for (var checkId : activeChecks) { try { - Check check = checksRegistry.getExtension(checkId).orElseThrow( + var check = checksRegistry.getExtension(checkId).orElseThrow( () -> new NoSuchElementException("Unknown check [" + checkId + "]")); - long tStartTask = currentTimeMillis(); + var tStartTask = currentTimeMillis(); LOG.debug("CasDoctor analysis [" + check.getId() + "] running..."); - ok &= check.check(aProject, aCas, aMessages); + ok &= check.check(aDocument, aDataOwner, aCas, aMessages); LOG.debug("CasDoctor analysis [" + check.getId() + "] completed in " + (currentTimeMillis() - tStartTask) + "ms"); } @@ -194,7 +195,7 @@ public boolean analyze(Project aProject, CAS aCas, List aMessages, throw new CasDoctorException(aMessages); } - long duration = currentTimeMillis() - tStart; + var duration = currentTimeMillis() - tStart; LOG.debug("CasDoctor completed {} checks in {}ms", activeChecks.size(), duration); serverTiming("CasDoctor", "CasDoctor (analyze)", duration); @@ -215,7 +216,7 @@ public void setActiveRepairs(String... aActiveRepairs) public void onApplicationStartedEvent(ApplicationStartedEvent aEvent) { // When under development, automatically enable all checks. - String version = SettingsUtil.getVersionProperties().getProperty(SettingsUtil.PROP_VERSION); + var version = SettingsUtil.getVersionProperties().getProperty(SettingsUtil.PROP_VERSION); if ("unknown".equals(version) || version.contains("-SNAPSHOT") || version.contains("-beta-")) { if (disableAutoScan) { @@ -228,11 +229,11 @@ public void onApplicationStartedEvent(ApplicationStartedEvent aEvent) } } - for (String checkId : activeChecks) { + for (var checkId : activeChecks) { LOG.info("Check activated: " + checkId); } - for (String repairId : activeRepairs) { + for (var repairId : activeRepairs) { LOG.info("Repair activated: " + repairId); } } diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheck.java index c6ca92380f3..ac3180f8ed0 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheck.java @@ -30,7 +30,7 @@ import org.apache.uima.cas.CAS; import org.apache.uima.cas.Type; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; import de.tudarmstadt.ukp.inception.support.text.TrimUtils; @@ -46,13 +46,14 @@ public AllAnnotationsStartAndEndWithCharactersCheck(AnnotationSchemaService aAnn } @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { if (annotationService == null) { return true; } - var allAnnoLayers = annotationService.listAnnotationLayer(aProject); + var allAnnoLayers = annotationService.listAnnotationLayer(aDocument.getProject()); if (isEmpty(allAnnoLayers)) { return true; } diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheck.java index 211786c0263..34d4298ee32 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheck.java @@ -26,11 +26,9 @@ import org.apache.uima.cas.CAS; import org.apache.uima.cas.Type; -import org.apache.uima.cas.text.AnnotationFS; import org.springframework.util.CollectionUtils; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -46,20 +44,21 @@ public AllAnnotationsStartAndEndWithinSentencesCheck(AnnotationSchemaService aAn } @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { if (annotationService == null) { return true; } - List allAnnoLayers = annotationService.listAnnotationLayer(aProject); + var allAnnoLayers = annotationService.listAnnotationLayer(aDocument.getProject()); allAnnoLayers.removeIf(layer -> Sentence._TypeName.equals(layer.getName())); if (CollectionUtils.isEmpty(allAnnoLayers)) { return true; } boolean ok = true; - for (AnnotationLayer layer : allAnnoLayers) { + for (var layer : allAnnoLayers) { Type type; try { type = getType(aCas, layer.getName()); @@ -75,7 +74,7 @@ public boolean check(Project aProject, CAS aCas, List aMessages) continue; } - for (AnnotationFS ann : select(aCas, type)) { + for (var ann : select(aCas, type)) { var startsOutside = aCas.select(Sentence._TypeName) .covering(ann.getBegin(), ann.getBegin()).isEmpty(); var endsOutside = aCas.select(Sentence._TypeName) diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllFeatureStructuresIndexedCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllFeatureStructuresIndexedCheck.java index ea0822da05b..a63c3a10de6 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllFeatureStructuresIndexedCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllFeatureStructuresIndexedCheck.java @@ -26,14 +26,15 @@ import org.apache.uima.cas.CAS; import org.apache.uima.cas.FeatureStructure; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public class AllFeatureStructuresIndexedCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { Map nonIndexed = getNonIndexedFSesWithOwner(aCas); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/CASMetadataTypeIsPresentCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/CASMetadataTypeIsPresentCheck.java index 91b28953e8e..51f4473dfb9 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/CASMetadataTypeIsPresentCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/CASMetadataTypeIsPresentCheck.java @@ -22,7 +22,7 @@ import org.apache.uima.cas.CAS; import de.tudarmstadt.ukp.clarin.webanno.api.type.CASMetadata; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; /** @@ -33,7 +33,8 @@ public class CASMetadataTypeIsPresentCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { if (aCas.getTypeSystem().getType(CASMetadata._TypeName) == null) { aMessages.add(LogMessage.info(this, "CAS needs upgrade to support CASMetadata which is " diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/Check.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/Check.java index d6ca956ca67..e25f582230c 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/Check.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/Check.java @@ -21,14 +21,15 @@ import org.apache.uima.cas.CAS; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.extensionpoint.Extension; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public interface Check extends Extension { - boolean check(Project aProject, CAS aCas, List aMessages); + boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages); @Override default String getId() diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java index 163f76959ea..eed28e1ece2 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java @@ -25,11 +25,9 @@ import org.apache.uima.cas.CAS; import org.apache.uima.cas.Feature; -import org.apache.uima.cas.FeatureStructure; -import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.layer.relation.RelationAdapter; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -50,23 +48,24 @@ public DanglingRelationsCheck(AnnotationSchemaService aAnnotationService) } @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { boolean ok = true; - for (AnnotationFS fs : aCas.getAnnotationIndex()) { - Type t = fs.getType(); + for (var fs : aCas.getAnnotationIndex()) { + var t = fs.getType(); - Feature sourceFeat = t.getFeatureByBaseName(FEAT_REL_SOURCE); - Feature targetFeat = t.getFeatureByBaseName(FEAT_REL_TARGET); + var sourceFeat = t.getFeatureByBaseName(FEAT_REL_SOURCE); + var targetFeat = t.getFeatureByBaseName(FEAT_REL_TARGET); // Is this a relation? if (!(sourceFeat != null && targetFeat != null)) { continue; } - RelationAdapter relationAdapter = (RelationAdapter) annotationService - .findAdapter(aProject, fs); + var relationAdapter = (RelationAdapter) annotationService + .findAdapter(aDocument.getProject(), fs); Feature relationSourceAttachFeature = null; Feature relationTargetAttachFeature = null; @@ -77,8 +76,8 @@ public boolean check(Project aProject, CAS aCas, List aMessages) .getFeatureByBaseName(relationAdapter.getAttachFeatureName()); } - FeatureStructure source = fs.getFeatureValue(sourceFeat); - FeatureStructure target = fs.getFeatureValue(targetFeat); + var source = fs.getFeatureValue(sourceFeat); + var target = fs.getFeatureValue(targetFeat); // Here we get the annotations that the relation is pointing to in the UI if (source != null && relationSourceAttachFeature != null) { @@ -91,7 +90,7 @@ public boolean check(Project aProject, CAS aCas, List aMessages) // Does it have null endpoints? if (source == null || target == null) { - StringBuilder message = new StringBuilder(); + var message = new StringBuilder(); message.append("Relation [" + relationAdapter.getLayer().getName() + "] with id [" + ICasUtil.getAddr(fs) + "] has loose ends."); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DocumentTextStartsWithBomCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DocumentTextStartsWithBomCheck.java index a01b346f854..88a59bba897 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DocumentTextStartsWithBomCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DocumentTextStartsWithBomCheck.java @@ -21,14 +21,15 @@ import org.apache.uima.cas.CAS; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public class DocumentTextStartsWithBomCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { // BOM for UTF-16BE: FE FF // BOM for UTF-16LE: FF FE diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/FeatureAttachedSpanAnnotationsTrulyAttachedCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/FeatureAttachedSpanAnnotationsTrulyAttachedCheck.java index 2f7aff12fe3..0a74438230b 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/FeatureAttachedSpanAnnotationsTrulyAttachedCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/FeatureAttachedSpanAnnotationsTrulyAttachedCheck.java @@ -28,7 +28,7 @@ import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogLevel; @@ -48,11 +48,12 @@ public FeatureAttachedSpanAnnotationsTrulyAttachedCheck( } @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { - boolean ok = true; - int count = 0; - for (var layer : annotationService.listAnnotationLayer(aProject)) { + var ok = true; + var count = 0; + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!(SpanLayerSupport.TYPE.equals(layer.getType()) && layer.getAttachFeature() != null)) { continue; diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/LinksReachableThroughChainsCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/LinksReachableThroughChainsCheck.java index 742e78cc274..3483236188c 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/LinksReachableThroughChainsCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/LinksReachableThroughChainsCheck.java @@ -24,13 +24,11 @@ import java.util.List; import org.apache.uima.cas.CAS; -import org.apache.uima.cas.FeatureStructure; import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; import org.apache.uima.fit.util.FSUtil; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.WebAnnoConst; import de.tudarmstadt.ukp.inception.support.logging.LogLevel; @@ -47,10 +45,11 @@ public LinksReachableThroughChainsCheck(AnnotationSchemaService aAnnotationServi } @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { boolean ok = true; - for (AnnotationLayer layer : annotationService.listAnnotationLayer(aProject)) { + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!WebAnnoConst.CHAIN_TYPE.equals(layer.getType())) { continue; } @@ -72,8 +71,8 @@ public boolean check(Project aProject, CAS aCas, List aMessages) var chains = aCas.select(chainType).asList(); var links = new ArrayList<>(select(aCas, linkType)); - for (FeatureStructure chain : chains) { - AnnotationFS link = FSUtil.getFeature(chain, "first", AnnotationFS.class); + for (var chain : chains) { + var link = FSUtil.getFeature(chain, "first", AnnotationFS.class); while (link != null) { links.remove(link); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheck.java index 0974c11dd00..5506e93acbd 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheck.java @@ -24,14 +24,15 @@ import org.apache.uima.cas.CAS; import org.apache.uima.jcas.tcas.Annotation; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public class NegativeSizeAnnotationsCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { boolean ok = true; diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheck.java index 0ebe7fc9c95..bcb00af9780 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheck.java @@ -34,8 +34,7 @@ import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -52,19 +51,20 @@ public NoMultipleIncomingRelationsCheck(AnnotationSchemaService aAnnotationServi } @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { if (annotationService == null) { return true; } - List allAnnoLayers = annotationService.listAnnotationLayer(aProject); + var allAnnoLayers = annotationService.listAnnotationLayer(aDocument.getProject()); if (CollectionUtils.isEmpty(allAnnoLayers)) { return true; } boolean ok = true; - for (AnnotationLayer layer : allAnnoLayers) { + for (var layer : allAnnoLayers) { if (!RELATION_TYPE.equals(layer.getType())) { continue; @@ -89,14 +89,13 @@ public boolean check(Project aProject, CAS aCas, List aMessages) // to provide a better debugging output. Map incoming = new HashMap<>(); - for (AnnotationFS rel : select(aCas, type)) { + for (var rel : select(aCas, type)) { - AnnotationFS source = getFeature(rel, FEAT_REL_SOURCE, AnnotationFS.class); - AnnotationFS target = getFeature(rel, FEAT_REL_TARGET, AnnotationFS.class); + var source = getFeature(rel, FEAT_REL_SOURCE, AnnotationFS.class); + var target = getFeature(rel, FEAT_REL_TARGET, AnnotationFS.class); - AnnotationFS existingSource = incoming.get(target); + var existingSource = incoming.get(target); if (existingSource != null) { - // Debug output should include sentence number to make the orientation // easier Optional sentenceNumber = Optional.empty(); @@ -118,7 +117,6 @@ public boolean check(Project aProject, CAS aCas, List aMessages) target.getCoveredText())); } else { - aMessages.add(LogMessage.warn(this, "Relation [%s] -> [%s] points to span that already has an " + "incoming relation [%s] -> [%s].", diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoZeroSizeTokensAndSentencesCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoZeroSizeTokensAndSentencesCheck.java index 28cc1d80ed3..104054ccc02 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoZeroSizeTokensAndSentencesCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoZeroSizeTokensAndSentencesCheck.java @@ -25,7 +25,7 @@ import org.apache.uima.cas.CAS; import org.apache.uima.cas.text.AnnotationFS; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogLevel; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -33,7 +33,8 @@ public class NoZeroSizeTokensAndSentencesCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { boolean ok = true; diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/RelationOffsetsCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/RelationOffsetsCheck.java index b0cef8c9b29..d94c2e47dd5 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/RelationOffsetsCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/RelationOffsetsCheck.java @@ -28,8 +28,7 @@ import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.WebAnnoConst; import de.tudarmstadt.ukp.inception.support.logging.LogLevel; @@ -51,11 +50,12 @@ public RelationOffsetsCheck(AnnotationSchemaService aAnnotationService) } @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { boolean ok = true; - for (AnnotationLayer layer : annotationService.listAnnotationLayer(aProject)) { + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!RELATION_TYPE.equals(layer.getType())) { continue; } @@ -70,9 +70,8 @@ public boolean check(Project aProject, CAS aCas, List aMessages) continue; } - for (AnnotationFS rel : select(aCas, type)) { - AnnotationFS target = getFeature(rel, WebAnnoConst.FEAT_REL_TARGET, - AnnotationFS.class); + for (var rel : select(aCas, type)) { + var target = getFeature(rel, WebAnnoConst.FEAT_REL_TARGET, AnnotationFS.class); if ((rel.getBegin() != target.getBegin()) || (rel.getEnd() != target.getEnd())) { aMessages.add(new LogMessage(this, LogLevel.ERROR, "Relation offsets [%d,%d] to not match target offsets [%d,%d]", diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheck.java index 0134f4c8877..86082fbbe4b 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheck.java @@ -25,7 +25,7 @@ import org.apache.uima.cas.CAS; import org.apache.uima.jcas.tcas.Annotation; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -34,7 +34,8 @@ public class TokensAndSententencedDoNotOverlapCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { return checkTokens(aCas, aMessages) && checkSentences(aCas, aMessages); } diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UniqueDocumentAnnotationCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UniqueDocumentAnnotationCheck.java index df6bbcd118a..be98577b75e 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UniqueDocumentAnnotationCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UniqueDocumentAnnotationCheck.java @@ -22,7 +22,7 @@ import org.apache.uima.cas.CAS; import org.apache.uima.jcas.tcas.DocumentAnnotation; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; /** @@ -32,7 +32,8 @@ public class UniqueDocumentAnnotationCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { if (aCas.select(DocumentAnnotation.class).count() > 1) { aMessages.add(LogMessage.error(this, "There is more than one document annotation!")); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UnreachableAnnotationsCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UnreachableAnnotationsCheck.java index 4ed9b0785d7..e8e599b4fb2 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UnreachableAnnotationsCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/UnreachableAnnotationsCheck.java @@ -34,7 +34,7 @@ import org.apache.uima.cas.impl.CASImpl; import org.apache.uima.resource.ResourceInitializationException; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; import de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil; @@ -42,7 +42,8 @@ public class UnreachableAnnotationsCheck implements Check { @Override - public boolean check(Project aProject, CAS aCas, List aMessages) + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { var casImpl = (CASImpl) getRealCas(aCas); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/XmlStructurePresentInCurationCasCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/XmlStructurePresentInCurationCasCheck.java new file mode 100644 index 00000000000..0e6663327ab --- /dev/null +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/XmlStructurePresentInCurationCasCheck.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.clarin.webanno.diag.checks; + +import static de.tudarmstadt.ukp.inception.support.WebAnnoConst.CURATION_USER; +import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; + +import java.io.IOException; +import java.util.List; + +import org.apache.uima.cas.CAS; +import org.dkpro.core.api.xml.type.XmlDocument; + +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.inception.documents.api.DocumentService; +import de.tudarmstadt.ukp.inception.support.logging.LogMessage; + +public class XmlStructurePresentInCurationCasCheck + implements Check +{ + private final DocumentService documentService; + + public XmlStructurePresentInCurationCasCheck(DocumentService aDocumentService) + { + documentService = aDocumentService; + } + + @Override + public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) + { + if (!CURATION_USER.equals(aDataOwner)) { + // We want to check only the curation CAS + return true; + } + + if (!aCas.select(XmlDocument.class).isEmpty()) { + // Document structure already exists + return true; + } + + try { + var initialCas = documentService.createOrReadInitialCas(aDocument); + + if (initialCas.select(XmlDocument.class).isEmpty()) { + // Initial CAS also does not contain a document structure, so we are good + return true; + } + + // If we get here, the curation CAS does not contain a document structure + // but the initial CAS did, so the structure is missing from the curation CAS. + aMessages.add(LogMessage.error(this, + "XML document structure that is present in the initial CAS has not been " + + "copied over to the curation CAS")); + return false; + } + catch (IOException e) { + aMessages.add(LogMessage.error(this, "Unable to obtain initial CAS: %s", + getRootCauseMessage(e))); + return false; + } + } +} diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/config/CasDoctorAutoConfiguration.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/config/CasDoctorAutoConfiguration.java index 3b4a4aab827..eae0e1f07ea 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/config/CasDoctorAutoConfiguration.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/config/CasDoctorAutoConfiguration.java @@ -46,6 +46,7 @@ import de.tudarmstadt.ukp.clarin.webanno.diag.checks.TokensAndSententencedDoNotOverlapCheck; import de.tudarmstadt.ukp.clarin.webanno.diag.checks.UniqueDocumentAnnotationCheck; import de.tudarmstadt.ukp.clarin.webanno.diag.checks.UnreachableAnnotationsCheck; +import de.tudarmstadt.ukp.clarin.webanno.diag.checks.XmlStructurePresentInCurationCasCheck; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.CoverAllTextInSentencesRepair; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.ReattachFeatureAttachedSpanAnnotationsAndDeleteExtrasRepair; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.ReattachFeatureAttachedSpanAnnotationsRepair; @@ -57,9 +58,11 @@ import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.RemoveDanglingRelationsRepair; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.RemoveZeroSizeTokensAndSentencesRepair; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair; +import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.ReplaceXmlStructureInCurationCasRepair; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.SwitchBeginAndEndOnNegativeSizedAnnotationsRepair; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.TrimAnnotationsRepair; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.UpgradeCasRepair; +import de.tudarmstadt.ukp.inception.documents.api.DocumentService; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @Configuration @@ -265,4 +268,18 @@ public RemoveBomRepair removeBomRepair() { return new RemoveBomRepair(); } + + @Bean + public XmlStructurePresentInCurationCasCheck xmlStructurePresentInCurationCasCheck( + DocumentService aDocumentService) + { + return new XmlStructurePresentInCurationCasCheck(aDocumentService); + } + + @Bean + public ReplaceXmlStructureInCurationCasRepair replaceXmlStructureInCurationCasRepair( + DocumentService aDocumentService) + { + return new ReplaceXmlStructureInCurationCasRepair(aDocumentService); + } } diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepair.java index 52dd95269e7..3e50859593e 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepair.java @@ -25,7 +25,7 @@ import org.apache.uima.cas.Type; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -34,7 +34,8 @@ public class CoverAllTextInSentencesRepair implements Repair { @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { int prevSentenceEnd = 0; for (Sentence sentence : aCas.select(Sentence.class)) { diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsAndDeleteExtrasRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsAndDeleteExtrasRepair.java index 649839b68bf..d950e01c250 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsAndDeleteExtrasRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsAndDeleteExtrasRepair.java @@ -27,12 +27,10 @@ import java.util.List; import org.apache.uima.cas.CAS; -import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -50,35 +48,36 @@ public ReattachFeatureAttachedSpanAnnotationsAndDeleteExtrasRepair( } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { - for (AnnotationLayer layer : annotationService.listAnnotationLayer(aProject)) { + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!(SpanLayerSupport.TYPE.equals(layer.getType()) && layer.getAttachFeature() != null)) { continue; } - Type attachType = getType(aCas, layer.getAttachType().getName()); - String attachFeature = layer.getAttachFeature().getName(); + var attachType = getType(aCas, layer.getAttachType().getName()); + var attachFeature = layer.getAttachFeature().getName(); - int count = 0; + var count = 0; // Go over the layer that has an attach feature (e.g. Token) and make sure that it is // filled // anno -> e.g. Lemma // attach -> e.g. Token - for (AnnotationFS anno : select(aCas, getType(aCas, layer.getName()))) { + for (var anno : select(aCas, getType(aCas, layer.getName()))) { // Here we fetch all annotations of the layer we attach to at the relevant position, // e.g. Token - List attachables = selectCovered(attachType, anno); + var attachables = selectCovered(attachType, anno); if (attachables.size() > 1) { aMessages.add(LogMessage.error(this, "There is more than one attachable annotation for [%s] on layer [%s].", layer.getName(), attachType.getName())); } - for (AnnotationFS attach : attachables) { - AnnotationFS existing = getFeature(attach, attachFeature, AnnotationFS.class); + for (var attach : attachables) { + var existing = getFeature(attach, attachFeature, AnnotationFS.class); // So there is an annotation to which we could attach and it does not yet have // an annotation attached, so we attach to it. @@ -102,17 +101,15 @@ public void repair(Project aProject, CAS aCas, List aMessages) // // attach -> e.g. Token // candidates -> e.g. Lemma - List toDelete = new ArrayList<>(); - for (AnnotationFS attach : select(aCas, attachType)) { - List candidates = selectCovered(getType(aCas, layer.getName()), - attach); + var toDelete = new ArrayList(); + for (var attach : select(aCas, attachType)) { + var candidates = selectCovered(getType(aCas, layer.getName()), attach); if (!candidates.isEmpty()) { // One of the candidates should already be attached - AnnotationFS attachedCandidate = getFeature(attach, attachFeature, - AnnotationFS.class); + var attachedCandidate = getFeature(attach, attachFeature, AnnotationFS.class); - for (AnnotationFS candidate : candidates) { + for (var candidate : candidates) { if (!candidate.equals(attachedCandidate)) { toDelete.add(candidate); } diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsRepair.java index 36a3177a0bb..af594ae9235 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReattachFeatureAttachedSpanAnnotationsRepair.java @@ -26,12 +26,10 @@ import java.util.List; import org.apache.uima.cas.CAS; -import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -48,30 +46,31 @@ public ReattachFeatureAttachedSpanAnnotationsRepair(AnnotationSchemaService aAnn } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { - for (AnnotationLayer layer : annotationService.listAnnotationLayer(aProject)) { + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!(SpanLayerSupport.TYPE.equals(layer.getType()) && layer.getAttachFeature() != null)) { continue; } - Type attachType = getType(aCas, layer.getAttachType().getName()); - String attachFeature = layer.getAttachFeature().getName(); + var attachType = getType(aCas, layer.getAttachType().getName()); + var attachFeature = layer.getAttachFeature().getName(); - int count = 0; - int nonNullCount = 0; + var count = 0; + var nonNullCount = 0; // Go over the layer that has an attach feature (e.g. Token) and make sure that it is // filled // anno -> e.g. Lemma // attach -> e.g. Token // Here we iterate over the attached layer, e.g. Lemma - for (AnnotationFS anno : select(aCas, getType(aCas, layer.getName()))) { + for (var anno : select(aCas, getType(aCas, layer.getName()))) { // Here we fetch all annotations of the layer we attach to at the relevant position, // e.g. Token - for (AnnotationFS attach : selectCovered(attachType, anno)) { - AnnotationFS existing = getFeature(attach, attachFeature, AnnotationFS.class); + for (var attach : selectCovered(attachType, anno)) { + var existing = getFeature(attach, attachFeature, AnnotationFS.class); if (existing == null) { setFeature(attach, layer.getAttachFeature().getName(), anno); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReindexFeatureAttachedSpanAnnotationsRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReindexFeatureAttachedSpanAnnotationsRepair.java index 000869bc800..82adf868c52 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReindexFeatureAttachedSpanAnnotationsRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReindexFeatureAttachedSpanAnnotationsRepair.java @@ -30,8 +30,7 @@ import org.apache.uima.cas.text.AnnotationFS; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogLevel; @@ -53,25 +52,25 @@ public ReindexFeatureAttachedSpanAnnotationsRepair(AnnotationSchemaService aAnno } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { Map nonIndexed = getNonIndexedFSesWithOwner(aCas); - for (AnnotationLayer layer : annotationService.listAnnotationLayer(aProject)) { + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!(SpanLayerSupport.TYPE.equals(layer.getType()) && layer.getAttachFeature() != null)) { continue; } - int count = 0; + var count = 0; // Go over the layer that has an attach feature (e.g. Token) and make sure that it is // filled // attach -> e.g. Token // anno -> e.g. Lemma - for (AnnotationFS attach : select(aCas, - getType(aCas, layer.getAttachType().getName()))) { - AnnotationFS anno = getFeature(attach, layer.getAttachFeature().getName(), + for (var attach : select(aCas, getType(aCas, layer.getAttachType().getName()))) { + var anno = getFeature(attach, layer.getAttachFeature().getName(), AnnotationFS.class); if (anno != null && nonIndexed.containsKey(anno)) { aCas.addFsToIndexes(anno); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RelationOffsetsRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RelationOffsetsRepair.java index 57a2ac6b2c7..655eea59db9 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RelationOffsetsRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RelationOffsetsRepair.java @@ -30,8 +30,7 @@ import org.apache.uima.cas.text.AnnotationFS; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.WebAnnoConst; import de.tudarmstadt.ukp.inception.support.logging.LogLevel; @@ -54,10 +53,11 @@ public RelationOffsetsRepair(AnnotationSchemaService aAnnotationService) } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { - List fixedRels = new ArrayList<>(); - for (AnnotationLayer layer : annotationService.listAnnotationLayer(aProject)) { + var fixedRels = new ArrayList(); + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!WebAnnoConst.RELATION_TYPE.equals(layer.getType())) { continue; } @@ -72,9 +72,8 @@ public void repair(Project aProject, CAS aCas, List aMessages) continue; } - for (AnnotationFS rel : select(aCas, type)) { - AnnotationFS target = getFeature(rel, WebAnnoConst.FEAT_REL_TARGET, - AnnotationFS.class); + for (var rel : select(aCas, type)) { + var target = getFeature(rel, WebAnnoConst.FEAT_REL_TARGET, AnnotationFS.class); if ((rel.getBegin() != target.getBegin()) || (rel.getEnd() != target.getEnd())) { fixedRels.add(rel); setFeature(rel, CAS.FEATURE_BASE_NAME_BEGIN, target.getBegin()); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepair.java index 6ef8630ebdd..db84b4518b6 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepair.java @@ -27,7 +27,7 @@ import org.apache.uima.jcas.tcas.Annotation; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @Safe(false) @@ -35,7 +35,8 @@ public class RemoveBomRepair implements Repair { @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { var cas = getRealCas(aCas); var text = cas.getDocumentText(); diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingChainLinksRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingChainLinksRepair.java index 9dc5599ca25..10c4b616ada 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingChainLinksRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingChainLinksRepair.java @@ -28,7 +28,7 @@ import org.apache.uima.jcas.tcas.Annotation; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.WebAnnoConst; import de.tudarmstadt.ukp.inception.support.logging.LogLevel; @@ -46,9 +46,10 @@ public RemoveDanglingChainLinksRepair(AnnotationSchemaService aAnnotationService } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { - for (var layer : annotationService.listAnnotationLayer(aProject)) { + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { if (!WebAnnoConst.CHAIN_TYPE.equals(layer.getType())) { continue; } diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingFeatureAttachedSpanAnnotationsRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingFeatureAttachedSpanAnnotationsRepair.java index 611d5cad021..3ab8aa0c5aa 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingFeatureAttachedSpanAnnotationsRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingFeatureAttachedSpanAnnotationsRepair.java @@ -30,7 +30,7 @@ import org.apache.uima.cas.text.AnnotationFS; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.layer.span.SpanLayerSupport; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -48,11 +48,12 @@ public RemoveDanglingFeatureAttachedSpanAnnotationsRepair( } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { var nonIndexed = getNonIndexedFSesWithOwner(aCas); - for (var layer : annotationService.listAnnotationLayer(aProject)) { + for (var layer : annotationService.listAnnotationLayer(aDocument.getProject())) { var count = 0; if (!(SpanLayerSupport.TYPE.equals(layer.getType()) diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepair.java index 2783d1cbeb6..7ce1d292074 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepair.java @@ -30,7 +30,7 @@ import org.apache.uima.cas.text.AnnotationFS; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.annotation.layer.relation.RelationAdapter; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.schema.api.adapter.TypeAdapter; @@ -58,7 +58,8 @@ public RemoveDanglingRelationsRepair(AnnotationSchemaService aAnnotationService) } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { var nonIndexed = getNonIndexedFSes(aCas); @@ -72,7 +73,7 @@ public void repair(Project aProject, CAS aCas, List aMessages) TypeAdapter adapter = null; try { adapter = adapterCache.computeIfAbsent(t.getName(), - $ -> annotationService.findAdapter(aProject, fs)); + $ -> annotationService.findAdapter(aDocument.getProject(), fs)); } catch (NoResultException e) { // Ignore diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveZeroSizeTokensAndSentencesRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveZeroSizeTokensAndSentencesRepair.java index a4d92709874..cdac9859516 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveZeroSizeTokensAndSentencesRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveZeroSizeTokensAndSentencesRepair.java @@ -27,7 +27,7 @@ import org.apache.uima.fit.util.FSUtil; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; import de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil; @@ -36,7 +36,8 @@ public class RemoveZeroSizeTokensAndSentencesRepair implements Repair { @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { for (AnnotationFS s : selectSentences(aCas)) { if (s.getBegin() >= s.getEnd()) { diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/Repair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/Repair.java index 1e635fdce46..16ac753a8b8 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/Repair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/Repair.java @@ -25,14 +25,14 @@ import org.apache.uima.cas.CAS; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.extensionpoint.Extension; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public interface Repair extends Extension { - void repair(Project aProject, CAS aCas, List aMessages); + void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, List aMessages); @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReplaceXmlStructureInCurationCasRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReplaceXmlStructureInCurationCasRepair.java new file mode 100644 index 00000000000..9280495f090 --- /dev/null +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/ReplaceXmlStructureInCurationCasRepair.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.clarin.webanno.diag.repairs; + +import static de.tudarmstadt.ukp.inception.io.xml.dkprocore.XmlNodeUtils.removeXmlDocumentStructure; +import static de.tudarmstadt.ukp.inception.io.xml.dkprocore.XmlNodeUtils.transferXmlDocumentStructure; +import static de.tudarmstadt.ukp.inception.support.WebAnnoConst.CURATION_USER; +import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; + +import java.io.IOException; +import java.util.List; + +import org.apache.uima.cas.CAS; +import org.dkpro.core.api.xml.type.XmlDocument; + +import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; +import de.tudarmstadt.ukp.inception.documents.api.DocumentService; +import de.tudarmstadt.ukp.inception.support.logging.LogMessage; + +@Safe(false) +public class ReplaceXmlStructureInCurationCasRepair + implements Repair +{ + private final DocumentService documentService; + + public ReplaceXmlStructureInCurationCasRepair(DocumentService aDocumentService) + { + documentService = aDocumentService; + } + + @Override + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) + { + if (!CURATION_USER.equals(aDataOwner)) { + // We want to repair only the curation CAS + return; + } + + try { + var initialCas = documentService.createOrReadInitialCas(aDocument); + + if (initialCas.select(XmlDocument.class).isEmpty()) { + // Initial CAS also does not contain a document structure, so we are good + return; + } + + var deleted = removeXmlDocumentStructure(aCas); + var added = transferXmlDocumentStructure(aCas, initialCas); + + var operation = "replaced"; + if (deleted == 0) { + operation = "added"; + } + + aMessages.add(LogMessage.error(this, + "XML document structure has been %s using the structure from the initial CAS (nodes: %d removed, %d added)", + operation, deleted, added)); + } + catch (IOException e) { + aMessages.add(LogMessage.error(this, "Unable to obtain initial CAS: %s", + getRootCauseMessage(e))); + } + } +} diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepair.java index 16d29fcc338..b5305798601 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepair.java @@ -25,7 +25,7 @@ import org.apache.uima.jcas.tcas.Annotation; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @Safe(false) @@ -33,7 +33,8 @@ public class SwitchBeginAndEndOnNegativeSizedAnnotationsRepair implements Repair { @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { for (Annotation ann : aCas.select(Annotation.class)) { if (ann.getBegin() > ann.getEnd()) { diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/TrimAnnotationsRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/TrimAnnotationsRepair.java index 32fe9568c80..50997b01e7a 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/TrimAnnotationsRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/TrimAnnotationsRepair.java @@ -31,7 +31,7 @@ import org.apache.uima.cas.Type; import org.apache.uima.jcas.tcas.Annotation; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; import de.tudarmstadt.ukp.inception.support.text.TrimUtils; @@ -47,9 +47,10 @@ public TrimAnnotationsRepair(AnnotationSchemaService aAnnotationService) } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { - var allAnnoLayers = annotationService.listAnnotationLayer(aProject); + var allAnnoLayers = annotationService.listAnnotationLayer(aDocument.getProject()); if (isEmpty(allAnnoLayers)) { return; } diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/UpgradeCasRepair.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/UpgradeCasRepair.java index 0fb9004ec13..1e9e64d51dc 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/UpgradeCasRepair.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/UpgradeCasRepair.java @@ -33,7 +33,7 @@ import org.slf4j.LoggerFactory; import de.tudarmstadt.ukp.clarin.webanno.diag.repairs.Repair.Safe; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; import de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil; @@ -56,7 +56,8 @@ public UpgradeCasRepair(AnnotationSchemaService aAnnotationService) } @Override - public void repair(Project aProject, CAS aCas, List aMessages) + public void repair(SourceDocument aDocument, String aDataOwner, CAS aCas, + List aMessages) { try { var casImpl = (CASImpl) getRealCas(aCas); @@ -65,7 +66,7 @@ public void repair(Project aProject, CAS aCas, List aMessages) var bytesBefore = size(serializeCASComplete(casImpl)); int bytesAfter; - annotationService.upgradeCas(aCas, aProject); + annotationService.upgradeCas(aCas, aDocument.getProject()); aMessages.add(LogMessage.info(this, "CAS upgraded.")); var annotationCountsAfter = countFeatureStructures(casImpl); diff --git a/inception/inception-diag/src/main/resources/META-INF/asciidoc/user-guide/casdoctor.adoc b/inception/inception-diag/src/main/resources/META-INF/asciidoc/user-guide/casdoctor.adoc index 7e3e40cb096..ea194b541b4 100644 --- a/inception/inception-diag/src/main/resources/META-INF/asciidoc/user-guide/casdoctor.adoc +++ b/inception/inception-diag/src/main/resources/META-INF/asciidoc/user-guide/casdoctor.adoc @@ -224,21 +224,29 @@ Removing them is harmless and reduces memory and disk space usage. [[check_AllAnnotationsStartAndEndWithCharactersCheck]] === All annotations start and end with characters [horizontal] -ID:: `check_AllAnnotationsStartAndEndWithCharactersCheck` +ID:: `AllAnnotationsStartAndEndWithCharactersCheck` Related repairs:: <> -Checks if all annotations start and end with a character (i.e. not a whitespace). Annotations that start or end with a -whitespace character can cause problems during rendering. Trimming whitespace at the begin and end is typically as -harmless procedure. +Checks if all annotations start and end with a character (i.e. not a whitespace). Annotations that start or end with a whitespace character can cause problems during rendering. +Trimming whitespace at the begin and end is typically as harmless procedure. [[check_DocumentTextStartsWithBomCheck]] === Document text starts with Byte Order Mark [horizontal] -ID:: `check_DocumentTextStartsWithBomCheck` +ID:: `DocumentTextStartsWithBomCheck` Related repairs:: <> Checks if the document text starts with a Byte Order Mark (BOM). +[[check_XmlStructurePresentInCurationCasCheck]] +=== XML structure is present in curation CAS +[horizontal] +ID:: `XmlStructurePresentInCurationCasCheck` +Related repairs:: <> + +Checks if an XML structure that may have been extracted from the source document is present in the curation CAS. +If it is not present, this check will fail. + [[sect_repairs]] == Repairs @@ -251,9 +259,8 @@ ID:: `ReattachFeatureAttachedSpanAnnotationsRepair` This repair action attempts to attach spans that should be attached to another span, but are not. E.g. it tries to set the `pos` feature of tokens to the POS annotation for that respective token. -The action is not performed if there are multiple stacked annotations to choose from. Stacked -attached annotations would be an indication of a bug because attached layers are not allowed to -stack. +The action is not performed if there are multiple stacked annotations to choose from. +Stacked attached annotations would be an indication of a bug because attached layers are not allowed to stack. This is a safe repair action as it does not delete anything. @@ -265,10 +272,10 @@ This is a safe repair action as it does not delete anything. ID:: `ReattachFeatureAttachedSpanAnnotationsAndDeleteExtrasRepair` This is a destructive variant of <>. In -addition to re-attaching unattached annotations, it also removes all extra candidates that cannot -be attached. For example, if there are two unattached Lemma annotations at the position of a Token -annotation, then one will be attached and the other will be deleted. Which one is attached and -which one is deleted is undefined. +addition to re-attaching unattached annotations, it also removes all extra candidates that cannot be attached. +For example, if there are two unattached Lemma annotations at the position of a Token +annotation, then one will be attached and the other will be deleted. +Which one is attached and which one is deleted is undefined. [[repair_ReindexFeatureAttachedSpanAnnotationsRepair]] @@ -277,8 +284,8 @@ which one is deleted is undefined. [horizontal] ID:: `ReindexFeatureAttachedSpanAnnotationsRepair` -This repair locates annotations that are reachable via a attach feature but which are not actually -indexed in the CAS. Such annotations are then added back to the CAS indexes. +This repair locates annotations that are reachable via a attach feature but which are not actually indexed in the CAS. +Such annotations are then added back to the CAS indexes. This is a safe repair action as it does not delete anything. @@ -289,9 +296,8 @@ This is a safe repair action as it does not delete anything. [horizontal] ID:: `RelationOffsetsRepair` -Fixes that the offsets of relations match the target of the relation. This mirrors the DKPro -Core convention that the offsets of a dependency relation must match the offsets of the -dependent. +Fixes that the offsets of relations match the target of the relation. +This mirrors the DKPro Core convention that the offsets of a dependency relation must match the offsets of the dependent. [[repair_RemoveDanglingChainLinksRepair]] @@ -408,3 +414,11 @@ NOTE: Run the checks again after applying this repair as certain annotations can ID:: `RemoveBomRepair` This repair removes the Byte Order Mark at the start of the document and adjusts all annotation offsets accordingly. + +[[repair_ReplaceXmlStructureInCurationCasRepair]] +=== Relace XML structure in the curation CAS + +[horizontal] +ID:: `ReplaceXmlStructureInCurationCasRepair` + +This repair ensures the XML document structure that may have been extracted from the source document is also present in the curation CAS. Any potentially existing XML document structure int he curation CAS will be removed and replaced with the structure from the source document. diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsIndexedCheckTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsIndexedCheckTest.java index d4f44b531c6..067f777bb4b 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsIndexedCheckTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsIndexedCheckTest.java @@ -79,7 +79,7 @@ public void testFail() throws Exception .toArray(String[]::new)); // A project is not required for this check - var result = cd.analyze(null, cas, messages); + var result = cd.analyze(null, null, cas, messages); messages.forEach($ -> LOG.debug("{}", $)); @@ -126,7 +126,7 @@ public void testOK() throws Exception .toArray(String[]::new)); // A project is not required for this check - var result = cd.analyze(null, cas, messages); + var result = cd.analyze(null, null, cas, messages); messages.forEach($ -> LOG.debug("{}", $)); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheckTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheckTest.java index 1bb1f383a82..e82a52cfa0c 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheckTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithCharactersCheckTest.java @@ -39,6 +39,7 @@ import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @@ -60,13 +61,18 @@ static class Config AllAnnotationsStartAndEndWithCharactersCheck sut; Project project; + SourceDocument document; + String dataOwner; JCas jCas; List layers; @BeforeEach void setup() throws Exception { - project = new Project(); + project = Project.builder().build(); + document = SourceDocument.builder() // + .withProject(project) // + .build(); jCas = JCasFactory.createJCas(); var namedEntityLayer = new AnnotationLayer(); @@ -88,7 +94,7 @@ void test() var messages = new ArrayList(); - var result = sut.check(project, jCas.getCas(), messages); + var result = sut.check(document, dataOwner, jCas.getCas(), messages); assertThat(result).isFalse(); assertThat(messages).hasSize(1); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheckTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheckTest.java index 9446139fc17..3ffb33afb7b 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheckTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/AllAnnotationsStartAndEndWithinSentencesCheckTest.java @@ -38,6 +38,7 @@ import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @@ -59,16 +60,21 @@ static class Config AllAnnotationsStartAndEndWithinSentencesCheck sut; Project project; + SourceDocument document; + String dataOwner; JCas jCas; List layers; @BeforeEach void setup() throws Exception { - project = new Project(); + project = Project.builder().build(); + document = SourceDocument.builder() // + .withProject(project) // + .build(); jCas = JCasFactory.createJCas(); - AnnotationLayer namedEntityLayer = new AnnotationLayer(); + var namedEntityLayer = new AnnotationLayer(); namedEntityLayer.setName(NamedEntity._TypeName); layers = asList(namedEntityLayer); } @@ -88,7 +94,7 @@ void test() var messages = new ArrayList(); - var result = sut.check(project, jCas.getCas(), messages); + var result = sut.check(document, dataOwner, jCas.getCas(), messages); assertThat(result).isFalse(); assertThat(messages).hasSize(1); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheckTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheckTest.java index 2a582df5794..d7f06c9f435 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheckTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NegativeSizeAnnotationsCheckTest.java @@ -28,20 +28,21 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; class NegativeSizeAnnotationsCheckTest { NegativeSizeAnnotationsCheck sut; - Project project; + SourceDocument document; + String dataOwner; JCas jCas; @BeforeEach void setup() throws Exception { sut = new NegativeSizeAnnotationsCheck(); - project = new Project(); + document = SourceDocument.builder().build(); jCas = JCasFactory.createJCas(); } @@ -56,7 +57,7 @@ void test() var messages = new ArrayList(); - var result = sut.check(project, jCas.getCas(), messages); + var result = sut.check(document, dataOwner, jCas.getCas(), messages); assertThat(result).isFalse(); assertThat(messages).hasSize(1); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheckTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheckTest.java index c9ed2d9ef9f..ef337efa2a4 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheckTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/NoMultipleIncomingRelationsCheckTest.java @@ -17,16 +17,15 @@ */ package de.tudarmstadt.ukp.clarin.webanno.diag.checks; +import static java.util.Arrays.asList; +import static org.apache.uima.fit.factory.JCasFactory.createJCas; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.when; import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.jcas.JCas; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mockito; @@ -37,11 +36,14 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer; +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.coref.type.CoreferenceChain; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; +import de.tudarmstadt.ukp.inception.annotation.layer.chain.ChainLayerSupport; +import de.tudarmstadt.ukp.inception.annotation.layer.relation.RelationLayerSupport; import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; -import de.tudarmstadt.ukp.inception.support.WebAnnoConst; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @ExtendWith(SpringExtension.class) @@ -59,42 +61,54 @@ static class Config @Autowired NoMultipleIncomingRelationsCheck sut; + Project project; + SourceDocument document; + String dataOwner; + + @BeforeEach + void setup() throws Exception + { + project = Project.builder().build(); + document = SourceDocument.builder() // + .withProject(project) // + .build(); + } + @Test public void testFail() throws Exception { - AnnotationLayer relationLayer = new AnnotationLayer(); + var relationLayer = new AnnotationLayer(); relationLayer.setName(Dependency.class.getName()); - relationLayer.setType(WebAnnoConst.RELATION_TYPE); - when(annotationService.listAnnotationLayer(Mockito.isNull())) - .thenReturn(Arrays.asList(relationLayer)); + relationLayer.setType(RelationLayerSupport.TYPE); + when(annotationService.listAnnotationLayer(project)).thenReturn(asList(relationLayer)); - JCas jcas = JCasFactory.createJCas(); + var jcas = createJCas(); jcas.setDocumentText("This is a test."); - Token spanThis = new Token(jcas, 0, 4); + var spanThis = new Token(jcas, 0, 4); spanThis.addToIndexes(); - Token spanIs = new Token(jcas, 5, 7); + var spanIs = new Token(jcas, 5, 7); spanIs.addToIndexes(); - Token spanA = new Token(jcas, 8, 9); + var spanA = new Token(jcas, 8, 9); spanA.addToIndexes(); - Dependency dep1 = new Dependency(jcas, 0, 7); + var dep1 = new Dependency(jcas, 0, 7); dep1.setGovernor(spanThis); dep1.setDependent(spanIs); dep1.addToIndexes(); - Dependency dep2 = new Dependency(jcas, 0, 9); + var dep2 = new Dependency(jcas, 0, 9); dep2.setGovernor(spanA); dep2.setDependent(spanIs); dep2.addToIndexes(); - List messages = new ArrayList<>(); + var messages = new ArrayList(); - boolean result = sut.check(null, jcas.getCas(), messages); + var result = sut.check(document, dataOwner, jcas.getCas(), messages); messages.forEach(System.out::println); @@ -111,39 +125,38 @@ public void testFail() throws Exception @Test public void testOK() throws Exception { - AnnotationLayer relationLayer = new AnnotationLayer(); + var relationLayer = new AnnotationLayer(); relationLayer.setName(Dependency.class.getName()); + relationLayer.setType(RelationLayerSupport.TYPE); + when(annotationService.listAnnotationLayer(Mockito.isNull())) + .thenReturn(asList(relationLayer)); - relationLayer.setType(WebAnnoConst.RELATION_TYPE); - Mockito.when(annotationService.listAnnotationLayer(Mockito.isNull())) - .thenReturn(Arrays.asList(relationLayer)); - - JCas jcas = JCasFactory.createJCas(); + var jcas = createJCas(); jcas.setDocumentText("This is a test."); - Token spanThis = new Token(jcas, 0, 4); + var spanThis = new Token(jcas, 0, 4); spanThis.addToIndexes(); - Token spanIs = new Token(jcas, 6, 8); + var spanIs = new Token(jcas, 6, 8); spanIs.addToIndexes(); - Token spanA = new Token(jcas, 9, 10); + var spanA = new Token(jcas, 9, 10); spanA.addToIndexes(); - Dependency dep1 = new Dependency(jcas, 0, 8); + var dep1 = new Dependency(jcas, 0, 8); dep1.setGovernor(spanThis); dep1.setDependent(spanIs); dep1.addToIndexes(); - Dependency dep2 = new Dependency(jcas, 6, 10); + var dep2 = new Dependency(jcas, 6, 10); dep2.setGovernor(spanIs); dep2.setDependent(spanA); dep2.addToIndexes(); - List messages = new ArrayList<>(); + var messages = new ArrayList(); - boolean result = sut.check(null, jcas.getCas(), messages); + var result = sut.check(document, dataOwner, jcas.getCas(), messages); messages.forEach(System.out::println); @@ -154,39 +167,38 @@ public void testOK() throws Exception public void testOkBecauseCoref() throws Exception { - AnnotationLayer relationLayer = new AnnotationLayer(); + var relationLayer = new AnnotationLayer(); relationLayer.setName(CoreferenceChain.class.getName()); + relationLayer.setType(ChainLayerSupport.TYPE); + when(annotationService.listAnnotationLayer(Mockito.isNull())) + .thenReturn(asList(relationLayer)); - relationLayer.setType(WebAnnoConst.CHAIN_TYPE); - Mockito.when(annotationService.listAnnotationLayer(Mockito.isNull())) - .thenReturn(Arrays.asList(relationLayer)); - - JCas jcas = JCasFactory.createJCas(); + var jcas = createJCas(); jcas.setDocumentText("This is a test."); - Token spanThis = new Token(jcas, 0, 4); + var spanThis = new Token(jcas, 0, 4); spanThis.addToIndexes(); - Token spanIs = new Token(jcas, 6, 8); + var spanIs = new Token(jcas, 6, 8); spanIs.addToIndexes(); - Token spanA = new Token(jcas, 9, 10); + var spanA = new Token(jcas, 9, 10); spanA.addToIndexes(); - Dependency dep1 = new Dependency(jcas, 0, 8); + var dep1 = new Dependency(jcas, 0, 8); dep1.setGovernor(spanThis); dep1.setDependent(spanIs); dep1.addToIndexes(); - Dependency dep2 = new Dependency(jcas, 0, 10); + var dep2 = new Dependency(jcas, 0, 10); dep2.setGovernor(spanA); dep2.setDependent(spanIs); dep2.addToIndexes(); - List messages = new ArrayList<>(); + var messages = new ArrayList(); - boolean result = sut.check(null, jcas.getCas(), messages); + var result = sut.check(document, dataOwner, jcas.getCas(), messages); messages.forEach(System.out::println); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheckTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheckTest.java index 14a74c93d3f..9e35f9360b6 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheckTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/TokensAndSententencedDoNotOverlapCheckTest.java @@ -28,7 +28,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; @@ -36,14 +36,15 @@ class TokensAndSententencedDoNotOverlapCheckTest { TokensAndSententencedDoNotOverlapCheck sut; - Project project; + SourceDocument document; + String dataOwner; JCas jCas; @BeforeEach void setup() throws Exception { sut = new TokensAndSententencedDoNotOverlapCheck(); - project = new Project(); + document = SourceDocument.builder().build(); jCas = JCasFactory.createJCas(); } @@ -60,7 +61,7 @@ void thatOverlappingSentencesFailCheck() var messages = new ArrayList(); - var result = sut.check(project, jCas.getCas(), messages); + var result = sut.check(document, dataOwner, jCas.getCas(), messages); assertThat(result).isFalse(); assertThat(messages).hasSize(1); @@ -81,7 +82,7 @@ void thatOverlappingTokensFailCheck() var messages = new ArrayList(); - var result = sut.check(project, jCas.getCas(), messages); + var result = sut.check(document, dataOwner, jCas.getCas(), messages); assertThat(result).isFalse(); assertThat(messages).hasSize(1); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepairTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepairTest.java index 365b627bb6c..2352c0ea4ae 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepairTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/CoverAllTextInSentencesRepairTest.java @@ -29,21 +29,22 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; class CoverAllTextInSentencesRepairTest { CoverAllTextInSentencesRepair sut; - Project project; + SourceDocument document; + String dataOwner; JCas jCas; @BeforeEach void setup() throws Exception { sut = new CoverAllTextInSentencesRepair(); - project = new Project(); + document = SourceDocument.builder().build(); jCas = JCasFactory.createJCas(); } @@ -59,7 +60,7 @@ void thatNewSentenceIsAddedOnText() var messages = new ArrayList(); - sut.repair(project, jCas.getCas(), messages); + sut.repair(document, dataOwner, jCas.getCas(), messages); assertThat(jCas.select(Sentence.class).asList()) // .extracting(Annotation::getBegin, Annotation::getEnd) @@ -81,7 +82,7 @@ void thatNoNewSentenceIsAddedOnBlank() var messages = new ArrayList(); - sut.repair(project, jCas.getCas(), messages); + sut.repair(document, dataOwner, jCas.getCas(), messages); assertThat(jCas.select(Sentence.class).asList()) // .extracting(Annotation::getBegin, Annotation::getEnd) @@ -101,7 +102,7 @@ void thatNewSentenceIsAddedAtDocumentStart() var messages = new ArrayList(); - sut.repair(project, jCas.getCas(), messages); + sut.repair(document, dataOwner, jCas.getCas(), messages); assertThat(jCas.select(Sentence.class).asList()) // .extracting(Annotation::getBegin, Annotation::getEnd) @@ -122,7 +123,7 @@ void thatNewSentenceIsAddedAtDocumentEnd() var messages = new ArrayList(); - sut.repair(project, jCas.getCas(), messages); + sut.repair(document, dataOwner, jCas.getCas(), messages); assertThat(jCas.select(Sentence.class).asList()) // .extracting(Annotation::getBegin, Annotation::getEnd) diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepairTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepairTest.java index 4d4aab9d4df..c6e2207b29d 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepairTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveBomRepairTest.java @@ -59,7 +59,7 @@ void test() throws Exception } var messages = new ArrayList(); - sut.repair(null, cas, messages); + sut.repair(null, null, cas, messages); assertThat(annotations).hasSizeGreaterThan(annotationCount); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepairTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepairTest.java index 2ad73144b15..798e5ca1203 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepairTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/RemoveDanglingRelationsRepairTest.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import org.apache.uima.fit.factory.JCasFactory; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -37,6 +38,8 @@ import de.tudarmstadt.ukp.clarin.webanno.diag.ChecksRegistryImpl; import de.tudarmstadt.ukp.clarin.webanno.diag.RepairsRegistryImpl; import de.tudarmstadt.ukp.clarin.webanno.diag.checks.AllFeatureStructuresIndexedCheck; +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.dependency.Dependency; import de.tudarmstadt.ukp.inception.annotation.layer.relation.RelationAdapter; @@ -49,6 +52,19 @@ public class RemoveDanglingRelationsRepairTest private @Mock ConstraintsService constraintsService; private @Mock AnnotationSchemaService schemaService; + Project project; + SourceDocument document; + String dataOwner; + + @BeforeEach + void setup() throws Exception + { + project = Project.builder().build(); + document = SourceDocument.builder() // + .withProject(project) // + .build(); + } + @Test public void test() throws Exception { @@ -84,10 +100,10 @@ public void test() throws Exception .toArray(String[]::new)); // A project is not required for this check - var result = cd.analyze(null, jcas.getCas(), messages); + var result = cd.analyze(null, null, jcas.getCas(), messages); // A project is not required for this repair - cd.repair(null, jcas.getCas(), messages); + cd.repair(document, dataOwner, jcas.getCas(), messages); assertFalse(result); diff --git a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepairTest.java b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepairTest.java index 20d34658544..a7319fc3a69 100644 --- a/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepairTest.java +++ b/inception/inception-diag/src/test/java/de/tudarmstadt/ukp/clarin/webanno/diag/repairs/SwitchBeginAndEndOnNegativeSizedAnnotationsRepairTest.java @@ -29,21 +29,18 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import de.tudarmstadt.ukp.clarin.webanno.model.Project; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; import de.tudarmstadt.ukp.inception.support.logging.LogMessage; class SwitchBeginAndEndOnNegativeSizedAnnotationsRepairTest { SwitchBeginAndEndOnNegativeSizedAnnotationsRepair sut; - Project project; JCas jCas; @BeforeEach void setup() throws Exception { sut = new SwitchBeginAndEndOnNegativeSizedAnnotationsRepair(); - project = new Project(); jCas = JCasFactory.createJCas(); } @@ -58,7 +55,7 @@ void test() var messages = new ArrayList(); - sut.repair(project, jCas.getCas(), messages); + sut.repair(null, null, jCas.getCas(), messages); assertThat(jCas.select(Token.class).asList()) // .extracting(Annotation::getBegin, Annotation::getEnd) diff --git a/inception/inception-export/src/main/java/de/tudarmstadt/ukp/inception/export/DocumentImportExportServiceImpl.java b/inception/inception-export/src/main/java/de/tudarmstadt/ukp/inception/export/DocumentImportExportServiceImpl.java index 2fb672975db..1cad67806fc 100644 --- a/inception/inception-export/src/main/java/de/tudarmstadt/ukp/inception/export/DocumentImportExportServiceImpl.java +++ b/inception/inception-export/src/main/java/de/tudarmstadt/ukp/inception/export/DocumentImportExportServiceImpl.java @@ -21,6 +21,7 @@ import static de.tudarmstadt.ukp.clarin.webanno.api.casstorage.CasAccessMode.UNMANAGED_ACCESS; import static de.tudarmstadt.ukp.inception.project.api.ProjectService.withProjectLogger; import static de.tudarmstadt.ukp.inception.support.WebAnnoConst.CURATION_USER; +import static de.tudarmstadt.ukp.inception.support.WebAnnoConst.INITIAL_CAS_PSEUDO_USER; import static de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil.exists; import static de.tudarmstadt.ukp.inception.support.uima.WebAnnoCasUtil.getRealCas; import static java.util.Arrays.asList; @@ -363,11 +364,10 @@ private void runCasDoctorOnImport(SourceDocument aDocument, FormatSupport aForma var casDoctor = new CasDoctor(checksRegistry, repairsRegistry); casDoctor.setActiveChecks( checksRegistry.getExtensions().stream().map(c -> c.getId()).toArray(String[]::new)); - casDoctor.analyze(aDocument.getProject(), aCas, messages, true); + casDoctor.analyze(aDocument, INITIAL_CAS_PSEUDO_USER, aCas, messages, true); } - private void splitTokens(CAS cas, FormatSupport aFormat) - throws IOException + private void splitTokens(CAS cas, FormatSupport aFormat) throws IOException { var tokenType = getType(cas, Token.class); @@ -396,8 +396,7 @@ private void checkTokenQuota(CAS cas, FormatSupport aFormat) throws IOException } } - private void splitSenencesIfNecssary(CAS cas, FormatSupport aFormat) - throws IOException + private void splitSenencesIfNecssary(CAS cas, FormatSupport aFormat) throws IOException { var sentenceType = getType(cas, Sentence.class); diff --git a/inception/inception-io-xml/src/main/java/de/tudarmstadt/ukp/inception/io/xml/dkprocore/XmlNodeUtils.java b/inception/inception-io-xml/src/main/java/de/tudarmstadt/ukp/inception/io/xml/dkprocore/XmlNodeUtils.java index b2809d3e3d8..18eabf09230 100644 --- a/inception/inception-io-xml/src/main/java/de/tudarmstadt/ukp/inception/io/xml/dkprocore/XmlNodeUtils.java +++ b/inception/inception-io-xml/src/main/java/de/tudarmstadt/ukp/inception/io/xml/dkprocore/XmlNodeUtils.java @@ -33,16 +33,19 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.sax.SAXTransformerFactory; -import javax.xml.transform.sax.TransformerHandler; import javax.xml.transform.stream.StreamResult; +import org.apache.uima.cas.CAS; +import org.apache.uima.cas.FeatureStructure; import org.apache.uima.jcas.JCas; +import org.apache.uima.util.CasCopier; import org.dkpro.core.api.xml.type.XmlAttribute; +import org.dkpro.core.api.xml.type.XmlDocument; import org.dkpro.core.api.xml.type.XmlElement; import org.dkpro.core.api.xml.type.XmlNode; import org.dkpro.core.api.xml.type.XmlTextNode; @@ -61,8 +64,8 @@ public static boolean hasAttributeWithValue(XmlElement e, String aAttribute, Str static String serializeCasToXmlString(JCas aJCas) throws IOException { try (var out = new StringWriter()) { - SAXTransformerFactory tf = XmlParserUtils.newTransformerFactory(); - TransformerHandler th = tf.newTransformerHandler(); + var tf = XmlParserUtils.newTransformerFactory(); + var th = tf.newTransformerHandler(); th.getTransformer().setOutputProperty(OMIT_XML_DECLARATION, "yes"); th.getTransformer().setOutputProperty(METHOD, "xml"); th.getTransformer().setOutputProperty(INDENT, "no"); @@ -87,7 +90,7 @@ static void parseXmlStringToCas(JCas aJCas, String aXml) throws IOException static void parseXmlToCas(JCas aJCas, InputSource aSource) throws IOException { - CasXmlHandler handler = new CasXmlHandler(aJCas); + var handler = new CasXmlHandler(aJCas); try { var parser = XmlParserUtils.newSaxParser(); @@ -278,4 +281,30 @@ public static List rootPath(XmlElement aElement) return path; } + public static int transferXmlDocumentStructure(CAS aTarget, CAS aSource) + { + var copied = new AtomicInteger(); + var casCopier = new CasCopier(aSource, aTarget); + Consumer copyFunc = fs -> { + var copy = casCopier.copyFs(fs); + if (copy != null) { + copied.incrementAndGet(); + aTarget.addFsToIndexes(copy); + } + }; + + aSource.select(XmlDocument.class).forEach(copyFunc); + aSource.select(XmlNode.class).forEach(copyFunc); + return copied.get(); + } + + public static int removeXmlDocumentStructure(CAS aCas) + { + var toDelete = new ArrayList(); + aCas.select(XmlDocument.class).forEach(toDelete::add); + aCas.select(XmlNode.class).forEach(toDelete::add); + aCas.select(XmlAttribute.class).forEach(toDelete::add); + toDelete.forEach(aCas::removeFsFromIndexes); + return toDelete.size(); + } } diff --git a/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/CheckTask.java b/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/CheckTask.java index 858f3988a3e..8e2bbef5215 100644 --- a/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/CheckTask.java +++ b/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/CheckTask.java @@ -104,7 +104,8 @@ public void execute() casStorageService.forceActionOnCas(sd, INITIAL_CAS_PSEUDO_USER, (doc, user) -> createOrReadInitialCasWithoutSavingOrChecks(doc, messageSet), - (cas) -> casDoctor.analyze(project, cas, messageSet.getMessages()), // + (doc, user, cas) -> casDoctor.analyze(doc, user, cas, + messageSet.getMessages()), // false); } catch (Exception e) { @@ -126,7 +127,8 @@ public void execute() casStorageService.forceActionOnCas(sd, CURATION_USER, (doc, user) -> casStorageService.readCas(doc, user, UNMANAGED_NON_INITIALIZING_ACCESS), - (cas) -> casDoctor.analyze(project, cas, messageSet.getMessages()), // + (doc, user, cas) -> casDoctor.analyze(doc, user, cas, + messageSet.getMessages()), // false); } catch (FileNotFoundException e) { @@ -156,7 +158,8 @@ public void execute() casStorageService.forceActionOnCas(ad.getDocument(), ad.getUser(), (doc, user) -> casStorageService.readCas(doc, user, UNMANAGED_NON_INITIALIZING_ACCESS), - (cas) -> casDoctor.analyze(project, cas, messageSet.getMessages()), // + (doc, user, cas) -> casDoctor.analyze(doc, user, cas, + messageSet.getMessages()), // false); } } diff --git a/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/RepairTask.java b/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/RepairTask.java index 407b2a3746a..32fed7f8abf 100644 --- a/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/RepairTask.java +++ b/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/casdoctor/RepairTask.java @@ -104,7 +104,8 @@ public void execute() casStorageService.forceActionOnCas(sd, INITIAL_CAS_PSEUDO_USER, (doc, user) -> createOrReadInitialCasWithoutSavingOrChecks(doc, messageSet), - (cas) -> casDoctor.repair(project, cas, messageSet.getMessages()), // + (doc, user, cas) -> casDoctor.repair(doc, user, cas, + messageSet.getMessages()), // true); } catch (Exception e) { @@ -125,7 +126,8 @@ public void execute() casStorageService.forceActionOnCas(sd, CURATION_USER, (doc, user) -> casStorageService.readCas(doc, user, UNMANAGED_NON_INITIALIZING_ACCESS), - (cas) -> casDoctor.repair(project, cas, messageSet.getMessages()), // + (doc, user, cas) -> casDoctor.repair(doc, user, cas, + messageSet.getMessages()), // true); } catch (FileNotFoundException e) { @@ -158,7 +160,8 @@ public void execute() casStorageService.forceActionOnCas(sd, ad.getUser(), (doc, user) -> casStorageService.readCas(doc, user, UNMANAGED_NON_INITIALIZING_ACCESS), - (cas) -> casDoctor.repair(project, cas, messageSet.getMessages()), // + (doc, user, cas) -> casDoctor.repair(doc, user, cas, + messageSet.getMessages()), // true); } } From 437e3ff9c8b832d33c494d5dd0f547ab34b77547 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 15 Oct 2024 10:21:33 +0200 Subject: [PATCH 008/106] [maven-release-plugin] prepare release inception-34.1 --- inception/inception-active-learning/pom.xml | 2 +- inception/inception-agreement/pom.xml | 2 +- .../inception-annotation-storage-api/pom.xml | 2 +- .../inception-annotation-storage/pom.xml | 2 +- inception/inception-api-annotation/pom.xml | 2 +- inception/inception-api-editor/pom.xml | 2 +- inception/inception-api-formats/pom.xml | 2 +- inception/inception-api-render/pom.xml | 2 +- inception/inception-app-webapp/pom.xml | 2 +- inception/inception-bom/pom.xml | 254 +++++++++--------- inception/inception-boot-loader/pom.xml | 2 +- inception/inception-brat-editor/pom.xml | 2 +- inception/inception-build/pom.xml | 2 +- inception/inception-concept-linking/pom.xml | 2 +- inception/inception-constraints/pom.xml | 2 +- inception/inception-curation-legacy/pom.xml | 2 +- inception/inception-curation/pom.xml | 2 +- inception/inception-dependencies/pom.xml | 2 +- inception/inception-diag/pom.xml | 2 +- inception/inception-diam-api/pom.xml | 2 +- inception/inception-diam-compactv2/pom.xml | 2 +- inception/inception-diam-editor/pom.xml | 2 +- inception/inception-diam/pom.xml | 2 +- inception/inception-doc/pom.xml | 2 +- inception/inception-docker/pom.xml | 2 +- inception/inception-documents-api/pom.xml | 2 +- inception/inception-documents/pom.xml | 2 +- .../pom.xml | 2 +- inception/inception-export-api/pom.xml | 2 +- inception/inception-export/pom.xml | 2 +- inception/inception-external-editor/pom.xml | 2 +- .../inception-external-search-core/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../inception-external-search-pubmed/pom.xml | 2 +- .../inception-external-search-solr/pom.xml | 2 +- inception/inception-feature-lookup/pom.xml | 2 +- inception/inception-guidelines/pom.xml | 2 +- .../pom.xml | 2 +- inception/inception-html-editor/pom.xml | 2 +- .../inception-html-recogito-editor/pom.xml | 2 +- inception/inception-image/pom.xml | 2 +- .../inception-imls-azureai-openai/pom.xml | 2 +- inception/inception-imls-chatgpt/pom.xml | 2 +- inception/inception-imls-elg/pom.xml | 2 +- inception/inception-imls-external/pom.xml | 2 +- inception/inception-imls-hf/pom.xml | 2 +- inception/inception-imls-lapps/pom.xml | 2 +- inception/inception-imls-ollama/pom.xml | 2 +- inception/inception-imls-opennlp/pom.xml | 2 +- inception/inception-imls-stringmatch/pom.xml | 2 +- inception/inception-imls-support-llm/pom.xml | 2 +- inception/inception-imls-weblicht/pom.xml | 2 +- inception/inception-io-bioc/pom.xml | 2 +- inception/inception-io-brat/pom.xml | 2 +- inception/inception-io-conll/pom.xml | 2 +- inception/inception-io-html/pom.xml | 2 +- inception/inception-io-imscwb/pom.xml | 2 +- inception/inception-io-intertext/pom.xml | 2 +- inception/inception-io-json/pom.xml | 2 +- inception/inception-io-lif/pom.xml | 2 +- inception/inception-io-nif/pom.xml | 2 +- inception/inception-io-perseus/pom.xml | 2 +- inception/inception-io-rdf/pom.xml | 2 +- inception/inception-io-tcf/pom.xml | 2 +- inception/inception-io-tei/pom.xml | 2 +- inception/inception-io-text/pom.xml | 2 +- inception/inception-io-webanno-tsv/pom.xml | 2 +- inception/inception-io-xmi/pom.xml | 2 +- inception/inception-io-xml/pom.xml | 2 +- inception/inception-js-api/pom.xml | 2 +- inception/inception-kb-fact-linking/pom.xml | 2 +- inception/inception-kb-lucene-sail/pom.xml | 2 +- inception/inception-kb/pom.xml | 2 +- inception/inception-layer-docmetadata/pom.xml | 2 +- inception/inception-log-ui/pom.xml | 2 +- inception/inception-log/pom.xml | 2 +- inception/inception-model-export/pom.xml | 2 +- inception/inception-model-vdoc/pom.xml | 2 +- inception/inception-model/pom.xml | 2 +- inception/inception-pdf-editor/pom.xml | 2 +- inception/inception-pdf-editor2/pom.xml | 2 +- inception/inception-plugin-api/pom.xml | 2 +- inception/inception-plugin-manager/pom.xml | 2 +- inception/inception-plugin-parent/pom.xml | 2 +- inception/inception-preferences/pom.xml | 2 +- inception/inception-processing/pom.xml | 2 +- inception/inception-project-api/pom.xml | 2 +- inception/inception-project-export/pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../pom.xml | 2 +- .../inception-project-initializers-ud/pom.xml | 2 +- .../pom.xml | 2 +- .../inception-project-initializers/pom.xml | 2 +- inception/inception-project/pom.xml | 2 +- .../inception-recommendation-api/pom.xml | 2 +- inception/inception-recommendation/pom.xml | 2 +- inception/inception-remote/pom.xml | 2 +- inception/inception-review-editor/pom.xml | 2 +- inception/inception-scheduling/pom.xml | 2 +- inception/inception-schema-api/pom.xml | 2 +- inception/inception-schema/pom.xml | 2 +- inception/inception-search-core/pom.xml | 2 +- .../inception-search-mtas-upstream/pom.xml | 2 +- inception/inception-search-mtas/pom.xml | 2 +- inception/inception-security/pom.xml | 2 +- inception/inception-sharing/pom.xml | 2 +- inception/inception-support-bootstrap/pom.xml | 2 +- .../inception-support-standalone/pom.xml | 2 +- inception/inception-support/pom.xml | 2 +- inception/inception-telemetry/pom.xml | 2 +- inception/inception-test-dependencies/pom.xml | 2 +- inception/inception-testing/pom.xml | 2 +- inception/inception-tutorial/pom.xml | 2 +- inception/inception-ui-agreement/pom.xml | 2 +- inception/inception-ui-annotation/pom.xml | 2 +- inception/inception-ui-core/pom.xml | 2 +- inception/inception-ui-curation/pom.xml | 2 +- .../inception-ui-dashboard-activity/pom.xml | 2 +- inception/inception-ui-dashboard/pom.xml | 2 +- .../inception-ui-external-search/pom.xml | 2 +- inception/inception-ui-kb/pom.xml | 2 +- inception/inception-ui-project/pom.xml | 2 +- inception/inception-ui-scheduling/pom.xml | 2 +- inception/inception-ui-search/pom.xml | 2 +- inception/inception-ui-tagsets/pom.xml | 2 +- inception/inception-versioning/pom.xml | 2 +- inception/inception-websocket/pom.xml | 2 +- inception/inception-workload-dynamic/pom.xml | 2 +- inception/inception-workload-matrix/pom.xml | 2 +- inception/inception-workload-ui/pom.xml | 2 +- inception/inception-workload/pom.xml | 2 +- inception/pom.xml | 6 +- pom.xml | 4 +- 136 files changed, 265 insertions(+), 265 deletions(-) diff --git a/inception/inception-active-learning/pom.xml b/inception/inception-active-learning/pom.xml index a5cc0d8c11f..0ae438aa9e9 100644 --- a/inception/inception-active-learning/pom.xml +++ b/inception/inception-active-learning/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 .. inception-active-learning diff --git a/inception/inception-agreement/pom.xml b/inception/inception-agreement/pom.xml index 688f712606e..709551f528b 100644 --- a/inception/inception-agreement/pom.xml +++ b/inception/inception-agreement/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 inception-agreement diff --git a/inception/inception-annotation-storage-api/pom.xml b/inception/inception-annotation-storage-api/pom.xml index 961d76fff45..7924da5c92e 100644 --- a/inception/inception-annotation-storage-api/pom.xml +++ b/inception/inception-annotation-storage-api/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 inception-annotation-storage-api INCEpTION - Core - Annotation Storage - API diff --git a/inception/inception-annotation-storage/pom.xml b/inception/inception-annotation-storage/pom.xml index 88051bde133..0006ccfa035 100644 --- a/inception/inception-annotation-storage/pom.xml +++ b/inception/inception-annotation-storage/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 inception-annotation-storage INCEpTION - Core - Annotation Storage diff --git a/inception/inception-api-annotation/pom.xml b/inception/inception-api-annotation/pom.xml index 7203bea8abf..770bcf7b2ba 100644 --- a/inception/inception-api-annotation/pom.xml +++ b/inception/inception-api-annotation/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 inception-api-annotation INCEpTION - Core - Annotation API diff --git a/inception/inception-api-editor/pom.xml b/inception/inception-api-editor/pom.xml index abd27ceed38..fc007965831 100644 --- a/inception/inception-api-editor/pom.xml +++ b/inception/inception-api-editor/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 inception-api-editor INCEpTION - Core - Annotation editor API diff --git a/inception/inception-api-formats/pom.xml b/inception/inception-api-formats/pom.xml index 49e8e08a54b..2755068a6d2 100644 --- a/inception/inception-api-formats/pom.xml +++ b/inception/inception-api-formats/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 inception-api-formats INCEpTION - Core - Formats API diff --git a/inception/inception-api-render/pom.xml b/inception/inception-api-render/pom.xml index 1987d8ac534..10c6ba45111 100644 --- a/inception/inception-api-render/pom.xml +++ b/inception/inception-api-render/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 inception-api-render INCEpTION - Core - Annotation rendering API diff --git a/inception/inception-app-webapp/pom.xml b/inception/inception-app-webapp/pom.xml index d4e44ee57ab..78aba92c1cb 100644 --- a/inception/inception-app-webapp/pom.xml +++ b/inception/inception-app-webapp/pom.xml @@ -21,7 +21,7 @@ de.tudarmstadt.ukp.inception.app inception-app - 34.1-SNAPSHOT + 34.1 .. diff --git a/inception/inception-bom/pom.xml b/inception/inception-bom/pom.xml index 943b92d7c5a..719341d1ebd 100644 --- a/inception/inception-bom/pom.xml +++ b/inception/inception-bom/pom.xml @@ -20,7 +20,7 @@ de.tudarmstadt.ukp.inception.app inception - 34.1-SNAPSHOT + 34.1 ../../pom.xml @@ -33,237 +33,237 @@ de.tudarmstadt.ukp.inception.app inception-doc - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-agreement - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-agreement - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-annotation - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-project - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-tagsets - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-curation - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-curation - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-search - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-workload - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-workload-dynamic - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-workload-matrix - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-workload-ui - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-external-editor - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-external-search-core - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-external-search-pubannotation - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-external-search-pubmed - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-external-search-opensearch - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-external-search-solr - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-external-search - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-dashboard - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-dashboard-activity - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-kb - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-kb-fact-linking - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-kb-lucene-sail - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-html-apache-annotator-editor - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-html-editor - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-html-recogito-editor - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-pdf-editor - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-pdf-editor2 - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-recommendation - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-recommendation-api - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-review-editor - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-sharing - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-preferences - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-processing - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-guidelines - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-schema - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-active-learning - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-core - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-ui-kb - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-concept-linking - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-imls-azureai-openai - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-imls-chatgpt - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-imls-opennlp - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-imls-elg - 34.1-SNAPSHOT + 34.1 de.tudarmstadt.ukp.inception.app inception-imls-hf - 34.1-SNAPSHOT + 34.1 -

-
-
- - +
+
+
+
+ + +
-
- -
+
@@ -37,7 +37,7 @@
-
+
diff --git a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/feature/number/NumberFeatureTraitsEditor.java b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/feature/number/NumberFeatureTraitsEditor.java index d4945d09e28..77176ca8216 100644 --- a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/feature/number/NumberFeatureTraitsEditor.java +++ b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/feature/number/NumberFeatureTraitsEditor.java @@ -17,6 +17,8 @@ */ package de.tudarmstadt.ukp.inception.annotation.feature.number; +import static de.tudarmstadt.ukp.inception.annotation.feature.number.NumberFeatureTraits.EditorType.SPINNER; +import static de.tudarmstadt.ukp.inception.support.lambda.HtmlElementEvents.CHANGE_EVENT; import static de.tudarmstadt.ukp.inception.support.lambda.LambdaBehavior.visibleWhen; import java.io.Serializable; @@ -31,7 +33,6 @@ import org.apache.wicket.model.CompoundPropertyModel; import org.apache.wicket.model.IModel; import org.apache.wicket.model.Model; -import org.apache.wicket.model.PropertyModel; import org.apache.wicket.spring.injection.annot.SpringBean; import org.wicketstuff.jquery.core.Options; import org.wicketstuff.kendo.ui.form.NumberTextField; @@ -70,7 +71,7 @@ public NumberFeatureTraitsEditor(String aId, feature = aFeature; traits = Model.of(readTraits()); - Form form = new Form(MID_FORM, CompoundPropertyModel.of(traits)) + var form = new Form(MID_FORM, CompoundPropertyModel.of(traits)) { private static final long serialVersionUID = 4456748721289266655L; @@ -82,8 +83,6 @@ protected void onSubmit() } }; form.setOutputMarkupPlaceholderTag(true); - form.add(visibleWhen( - () -> traits.getObject().isLimited() && feature.getObject().getTagset() == null)); add(form); Class clazz = Integer.class; @@ -101,25 +100,30 @@ protected void onSubmit() } } - DropDownChoice editorType = new DropDownChoice<>( - CID_EDITOR_TYPE); - editorType.setModel(PropertyModel.of(traits, "editorType")); + var limited = new CheckBox("limited"); + limited.add(new LambdaAjaxFormComponentUpdatingBehavior(CHANGE_EVENT, + target -> target.add(form))); + form.add(limited); + + var editorType = new DropDownChoice(CID_EDITOR_TYPE); + // editorType.setModel(PropertyModel.of(traits, "editorType")); editorType.setChoices(Arrays.asList(NumberFeatureTraits.EditorType.values())); editorType.add(new LambdaAjaxFormComponentUpdatingBehavior("change")); - editorType.add(visibleWhen(() -> isEditorTypeSelectionPossible())); + editorType.add(visibleWhen( + () -> traits.getObject().isLimited() && isEditorTypeSelectionPossible())); form.add(editorType); var minimum = new NumberTextField<>("minimum", clazz, options); - minimum.setModel(PropertyModel.of(traits, "minimum")); + minimum.add(visibleWhen(() -> traits.getObject().isLimited())); form.add(minimum); var maximum = new NumberTextField<>("maximum", clazz, options); - maximum.setModel(PropertyModel.of(traits, "maximum")); + maximum.add(visibleWhen(() -> traits.getObject().isLimited())); form.add(maximum); minimum.add(new LambdaAjaxFormComponentUpdatingBehavior("change", target -> { - BigDecimal min = new BigDecimal(traits.getObject().getMinimum().toString()); - BigDecimal max = new BigDecimal(traits.getObject().getMaximum().toString()); + var min = new BigDecimal(traits.getObject().getMinimum().toString()); + var max = new BigDecimal(traits.getObject().getMaximum().toString()); if (min.compareTo(max) > 0) { traits.getObject().setMaximum(traits.getObject().getMinimum()); } @@ -127,19 +131,13 @@ protected void onSubmit() })); maximum.add(new LambdaAjaxFormComponentUpdatingBehavior("change", target -> { - BigDecimal min = new BigDecimal(traits.getObject().getMinimum().toString()); - BigDecimal max = new BigDecimal(traits.getObject().getMaximum().toString()); + var min = new BigDecimal(traits.getObject().getMinimum().toString()); + var max = new BigDecimal(traits.getObject().getMaximum().toString()); if (min.compareTo(max) > 0) { traits.getObject().setMinimum(traits.getObject().getMaximum()); } target.add(form); })); - - CheckBox multipleRows = new CheckBox("limited"); - multipleRows.setModel(PropertyModel.of(traits, "limited")); - multipleRows.add( - new LambdaAjaxFormComponentUpdatingBehavior("change", target -> target.add(form))); - add(multipleRows); } /** @@ -168,9 +166,9 @@ private UimaPrimitiveFeatureSupport_ImplBase getFeatureSupp */ private Traits readTraits() { - Traits result = new Traits(); + var result = new Traits(); - NumberFeatureTraits t = getFeatureSupport().readTraits(feature.getObject()); + var t = getFeatureSupport().readTraits(feature.getObject()); result.setLimited(t.isLimited()); result.setMinimum(t.getMinimum()); @@ -186,13 +184,13 @@ private Traits readTraits() */ private void writeTraits() { - NumberFeatureTraits t = new NumberFeatureTraits(); + var t = new NumberFeatureTraits(); t.setLimited(traits.getObject().isLimited()); t.setMinimum(traits.getObject().getMinimum()); t.setMaximum(traits.getObject().getMaximum()); - t.setEditorType(isEditorTypeSelectionPossible() ? traits.getObject().getEditorType() - : NumberFeatureTraits.EditorType.SPINNER); + t.setEditorType( + isEditorTypeSelectionPossible() ? traits.getObject().getEditorType() : SPINNER); getFeatureSupport().writeTraits(feature.getObject(), t); } @@ -216,9 +214,9 @@ public boolean isLimited() return limited; } - public void setLimited(boolean limited) + public void setLimited(boolean aLimited) { - this.limited = limited; + limited = aLimited; } public Number getMinimum() @@ -226,9 +224,9 @@ public Number getMinimum() return minimum; } - public void setMinimum(Number minimum) + public void setMinimum(Number aMinimum) { - this.minimum = minimum; + minimum = aMinimum; } public Number getMaximum() @@ -236,9 +234,9 @@ public Number getMaximum() return maximum; } - public void setMaximum(Number maximum) + public void setMaximum(Number aMaximum) { - this.maximum = maximum; + maximum = aMaximum; } public NumberFeatureTraits.EditorType getEditorType() @@ -246,9 +244,9 @@ public NumberFeatureTraits.EditorType getEditorType() return editorType; } - public void setEditorType(NumberFeatureTraits.EditorType editorType) + public void setEditorType(NumberFeatureTraits.EditorType aEditorType) { - this.editorType = editorType; + editorType = aEditorType; } } } diff --git a/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/layers/ProjectLayersPanel.html b/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/layers/ProjectLayersPanel.html index cd2a24eeb0f..bd17ea65701 100644 --- a/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/layers/ProjectLayersPanel.html +++ b/inception/inception-ui-project/src/main/java/de/tudarmstadt/ukp/clarin/webanno/ui/project/layers/ProjectLayersPanel.html @@ -156,6 +156,7 @@
+
Type cannot be changed after creation
@@ -284,6 +285,7 @@
+ -
- +
+ : +
-
-
-
-
From de38af11dfe9213eda4ad7d48829f2c4d14874ac Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 3 Nov 2024 13:24:31 +0100 Subject: [PATCH 031/106] #5121 - Improve recommender confusion matrix - Show column headers vertically to reduce skew in table --- ...html => ConfusionMatrixDialogContent.html} | 29 ++++++++++++++++--- ...java => ConfusionMatrixDialogContent.java} | 27 +++++++++-------- .../sidebar/RecommenderInfoPanel.java | 2 +- 3 files changed, 39 insertions(+), 19 deletions(-) rename inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/{RecommenderStatusDetailPanel.html => ConfusionMatrixDialogContent.html} (79%) rename inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/{RecommenderStatusDetailPanel.java => ConfusionMatrixDialogContent.java} (89%) diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.html b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/ConfusionMatrixDialogContent.html similarity index 79% rename from inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.html rename to inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/ConfusionMatrixDialogContent.html index 6203c7065d4..7f5beb30a04 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.html +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/ConfusionMatrixDialogContent.html @@ -17,6 +17,22 @@ limitations under the License. --> + + +
Confusion Matrix
@@ -54,11 +70,16 @@
Confusion Matrix
- - + + +
+ - - + + + + + diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/ConfusionMatrixDialogContent.java similarity index 89% rename from inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.java rename to inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/ConfusionMatrixDialogContent.java index c7db90098b7..729349edc49 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/ConfusionMatrixDialogContent.java @@ -34,21 +34,20 @@ import org.apache.wicket.markup.repeater.Item; import org.apache.wicket.markup.repeater.RefreshingView; import org.apache.wicket.model.IModel; -import org.apache.wicket.model.Model; import de.tudarmstadt.ukp.inception.recommendation.api.evaluation.ConfusionMatrix; import de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult; import de.tudarmstadt.ukp.inception.support.lambda.LambdaAjaxLink; import de.tudarmstadt.ukp.inception.support.wicket.DefaultRefreshingView; -public class RecommenderStatusDetailPanel +public class ConfusionMatrixDialogContent extends Panel { private static final long serialVersionUID = 6002385711741527179L; private RefreshingView rows; - public RecommenderStatusDetailPanel(String aId, IModel aModel) + public ConfusionMatrixDialogContent(String aId, IModel aModel) { super(aId, aModel); @@ -83,25 +82,25 @@ protected void populateItem(Item aCellItem) Fragment cell; // Header cell - if (aRowItem.getIndex() == 0) { - cell = new Fragment("cell", "th-centered", - RecommenderStatusDetailPanel.this); + if (aRowItem.getIndex() == 0 && aCellItem.getIndex() != 0) { + cell = new Fragment("cell", "th-column", + ConfusionMatrixDialogContent.this); } - else if (aCellItem.getIndex() == 0) { - cell = new Fragment("cell", "th-right", - RecommenderStatusDetailPanel.this); + else if (aRowItem.getIndex() != 0 && aCellItem.getIndex() == 0) { + cell = new Fragment("cell", "th-row", + ConfusionMatrixDialogContent.this); + } + else if (aRowItem.getIndex() == 0 && aCellItem.getIndex() == 0) { + cell = new Fragment("cell", "th", ConfusionMatrixDialogContent.this); } // Content cell else { - cell = new Fragment("cell", "td", RecommenderStatusDetailPanel.this); + cell = new Fragment("cell", "td", ConfusionMatrixDialogContent.this); } // Top-left cell - if (aRowItem.getIndex() == 0 && aCellItem.getIndex() == 0) { - cell.add(new Label("label", Model.of(""))); - } // Horizontal headers - else if (aRowItem.getIndex() == 0 && aCellItem.getIndex() != 0) { + if (aRowItem.getIndex() == 0 && aCellItem.getIndex() != 0) { cell.add(new Label("label", aCellItem.getModel())); } // Vertical headers diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java index cc45d09b3ca..324beb9f38c 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderInfoPanel.java @@ -243,7 +243,7 @@ public void onPredictionsSwitched(PredictionsSwitchedEvent aEvent) private void actionShowDetails(AjaxRequestTarget aTarget, Recommender aRecommender) { - var panel = new RecommenderStatusDetailPanel(ModalDialog.CONTENT_ID, + var panel = new ConfusionMatrixDialogContent(ModalDialog.CONTENT_ID, LoadableDetachableModel.of(() -> recommendationService .getEvaluatedRecommender(userService.getCurrentUser(), aRecommender).get()) .map(EvaluatedRecommender::getEvaluationResult)); From 4c991f5814086bb7b9a73b5d828cea4b62601642 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 3 Nov 2024 14:04:23 +0100 Subject: [PATCH 032/106] #5099 - Better support for auto-merging stacked annotations with link features - Add new setting to documentation - Allow actually setting the vote threshold to 0 - Minor refactoring --- .../projects_layers_feature_link.adoc | 6 ++ .../strategy/ThresholdBasedMergeStrategy.java | 5 ++ ...resholdBasedMergeStrategyTraitsEditor.java | 6 +- .../curation/merge/CasMergeLinkTest.java | 90 ++++++++++--------- 4 files changed, 63 insertions(+), 44 deletions(-) diff --git a/inception/inception-api-annotation/src/main/resources/META-INF/asciidoc/user-guide/projects_layers_feature_link.adoc b/inception/inception-api-annotation/src/main/resources/META-INF/asciidoc/user-guide/projects_layers_feature_link.adoc index e42b688e28f..7ce2abf940f 100644 --- a/inception/inception-api-annotation/src/main/resources/META-INF/asciidoc/user-guide/projects_layers_feature_link.adoc +++ b/inception/inception-api-annotation/src/main/resources/META-INF/asciidoc/user-guide/projects_layers_feature_link.adoc @@ -36,6 +36,12 @@ If the link feature is configured to any layer, an existing span annotation need If disabled the UI labels of annotations will be displayed instead of role labels. This property is enabled by default. +| Comparison +| Determines whether to consider this link feature when comparing the annotations of the layer to which this feature belongs. +By default, link featurse are **ignored when comparing annotations**. That means, two annotations at the same position with the same feature values but different links will be considered equal. +If you want to consider the links when comparing annotations, you can change this setting to **consider when comparing annotations**. +Note that even when the links are considered, two annotations at the same position are still considered to be stacked and will not be auto-merge by any merge strategies that do not support stacking. + | Multiplicity | Determines how links are compared to each other e.g. when calulating agreement or when merging annotations during curation. Use *Target can be linked in multiple different roles* if a link target can appear in multiple roles with respect to the same source span. diff --git a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategy.java b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategy.java index 1fc01c14b58..a776bc0d1b6 100644 --- a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategy.java +++ b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategy.java @@ -84,6 +84,11 @@ public List chooseConfigurationsToMerge(DiffResult aDiff, Configu return emptyList(); } + var totalAnnotators = cfgsAboveUserThreshold.stream() // + .flatMap(cfg -> cfg.getCasGroupIds().stream()) // + .distinct() // + .count(); + var totalVotes = cfgsAboveUserThreshold.stream() // .mapToDouble(cfg -> cfg.getCasGroupIds().size()) // .sum(); diff --git a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategyTraitsEditor.java b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategyTraitsEditor.java index 99a9a7ef430..e58f10e51b8 100644 --- a/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategyTraitsEditor.java +++ b/inception/inception-curation/src/main/java/de/tudarmstadt/ukp/inception/curation/merge/strategy/ThresholdBasedMergeStrategyTraitsEditor.java @@ -64,12 +64,14 @@ protected void onSubmit() form.add(new LambdaAjaxLink("presetUnanimousVote", this::actionPresetUnanimousVote)); - form.add(new NumberTextField<>("topRanks", Integer.class).setMinimum(1)); + form.add(new NumberTextField<>("topRanks", Integer.class).setMinimum(0)); form.add(new NumberTextField<>("userThreshold", Integer.class).setMinimum(1)); form.add(new NumberTextField<>("confidenceThreshold", Double.class) // - .setMinimum(0.0d).setMaximum(100.0d).setStep(0.1d) // + .setMinimum(0.0d) // + .setMaximum(100.0d) // + .setStep(0.1d) // .setModel(LambdaModelAdapter.of( // () -> traits.getConfidenceThreshold() * 100.0d, (v) -> traits.setConfidenceThreshold(v / 100.0d)))); diff --git a/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java b/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java index 50e6275ddaf..7462f9d38f8 100644 --- a/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java +++ b/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java @@ -43,6 +43,7 @@ import org.apache.uima.jcas.JCas; import org.apache.uima.jcas.tcas.Annotation; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -276,48 +277,53 @@ public void thatSecondLinkWithSameTargetAndSameRoleIsRejectedWhenRolesAreEnabled new LinkWithRoleModel("role1", null, targetFiller.getAddress())); } - @Test - public void thatStackedLinkHostsWithDifferentTargetsAreMerged() throws Exception + @Nested + class ThesholdBasedMergeStrategyTests { - slotLayer.setOverlapMode(ANY_OVERLAP); - var traits = new LinkFeatureTraits(); - traits.setDiffMode(INCLUDE); - slotHostDiffAdapter.addLinkFeature("links", "role", "target", ONE_TARGET_MULTIPLE_ROLES, - INCLUDE); - - slotFeature.setTraits(toJsonString(traits)); - sut.setMergeStrategy(ThresholdBasedMergeStrategy.builder() // - .withUserThreshold(1) // - .withConfidenceThreshold(0) // - .withTopRanks(Integer.MAX_VALUE) // - .build()); - - // Set up source CAS - buildAnnotation(sourceCas.getCas(), HOST_TYPE) // - .at(0, 0) // - .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role1", 1, 1))) - .buildAndAddToIndexes(); - - buildAnnotation(sourceCas.getCas(), HOST_TYPE) // - .at(0, 0) // - .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role2", 1, 1))) - .buildAndAddToIndexes(); - - var casMap = Map.of("source", sourceCas.getCas()); - - var diff = doDiff(diffAdapters, casMap).toResult(); - diff.print(System.out); - - sut.mergeCas(diff, document, DUMMY_USER, targetCas.getCas(), casMap); - - var targetHosts = targetCas.select(HOST_TYPE).asList(); - assertThat(targetHosts) // - .as("Links by host in target CAS") // - .hasSize(2) // - .extracting(host -> toMaterializedLinks(host, LINKS_FEATURE, "role", "target")) // - .containsExactlyInAnyOrder( // - asList(new MaterializedLink(LINKS_FEATURE, "role1", Token._TypeName, 1, 1)), // - asList(new MaterializedLink(LINKS_FEATURE, "role2", Token._TypeName, 1, - 1))); + @BeforeEach + void setup() throws Exception + { + slotLayer.setOverlapMode(ANY_OVERLAP); + var traits = new LinkFeatureTraits(); + traits.setDiffMode(INCLUDE); + slotHostDiffAdapter.addLinkFeature("links", "role", "target", ONE_TARGET_MULTIPLE_ROLES, + INCLUDE); + + slotFeature.setTraits(toJsonString(traits)); + sut.setMergeStrategy(ThresholdBasedMergeStrategy.builder() // + .withUserThreshold(1) // + .withConfidenceThreshold(0) // + .withTopRanks(Integer.MAX_VALUE) // + .build()); + } + + @Test + public void thatStackedLinkHostsWithDifferentTargetsAreMerged() throws Exception + { + buildAnnotation(sourceCas.getCas(), HOST_TYPE) // + .at(0, 0) // + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role1", 1, 1))) + .buildAndAddToIndexes(); + + buildAnnotation(sourceCas.getCas(), HOST_TYPE) // + .at(0, 0) // + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role2", 1, 1))) + .buildAndAddToIndexes(); + + var casMap = Map.of("source", sourceCas.getCas()); + var diff = doDiff(diffAdapters, casMap).toResult(); + sut.mergeCas(diff, document, DUMMY_USER, targetCas.getCas(), casMap); + + var targetHosts = targetCas.select(HOST_TYPE).asList(); + assertThat(targetHosts) // + .as("Links by host in target CAS") // + .hasSize(2) // + .extracting(host -> toMaterializedLinks(host, LINKS_FEATURE, "role", "target")) // + .containsExactlyInAnyOrder( // + asList(new MaterializedLink(LINKS_FEATURE, "role1", Token._TypeName, 1, + 1)), // + asList(new MaterializedLink(LINKS_FEATURE, "role2", Token._TypeName, 1, + 1))); + } } } From 5aeb412f4fa816ebf165f5a4258ffe4ed247d298 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 3 Nov 2024 14:25:03 +0100 Subject: [PATCH 033/106] #5099 - Better support for auto-merging stacked annotations with link features - Add two-user test --- .../curation/merge/CasMergeLinkTest.java | 95 +++++++++++++++---- 1 file changed, 75 insertions(+), 20 deletions(-) diff --git a/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java b/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java index 7462f9d38f8..2e3ba656650 100644 --- a/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java +++ b/inception/inception-curation/src/test/java/de/tudarmstadt/ukp/inception/curation/merge/CasMergeLinkTest.java @@ -61,7 +61,8 @@ public class CasMergeLinkTest private static final String DUMMY_USER = "dummyTargetUser"; - private JCas sourceCas; + private JCas sourceCas1; + private JCas sourceCas2; private JCas targetCas; @Override @@ -69,7 +70,8 @@ public class CasMergeLinkTest public void setup() throws Exception { super.setup(); - sourceCas = createJCas(); + sourceCas1 = createJCas(); + sourceCas2 = createJCas(); targetCas = createJCas(); } @@ -78,7 +80,7 @@ public void thatLinkIsCopiedFromSourceToTarget() throws Exception { // Set up source CAS var role = "slot1"; - var sourceFs = makeLinkHostFS(sourceCas, 0, 0, makeLinkFS(sourceCas, role, 0, 0)); + var sourceFs = makeLinkHostFS(sourceCas1, 0, 0, makeLinkFS(sourceCas1, role, 0, 0)); // Set up target CAS var target = makeLinkHostFS(targetCas, 0, 0); @@ -103,8 +105,8 @@ public void thatLinkIsAttachedToCorrectStackedTargetWithoutLabel() throws Except // Set up source CAS var role = "slot1"; - var sourceFs1 = makeLinkHostFS(sourceCas, 0, 0, makeLinkFS(sourceCas, role, 0, 0)); - var sourceFs2 = makeLinkHostFS(sourceCas, 0, 0, makeLinkFS(sourceCas, role, 1, 1)); + var sourceFs1 = makeLinkHostFS(sourceCas1, 0, 0, makeLinkFS(sourceCas1, role, 0, 0)); + var sourceFs2 = makeLinkHostFS(sourceCas1, 0, 0, makeLinkFS(sourceCas1, role, 1, 1)); // Set up target CAS var target1 = makeLinkHostFS(targetCas, 0, 0); @@ -144,9 +146,9 @@ public void thatLinkIsAttachedToCorrectStackedTargetWithLabel() throws Exception // Set up source CAS var role = "slot1"; - var sourceFs1 = makeLinkHostFS(sourceCas, 0, 0, makeLinkFS(sourceCas, role, 0, 0)); + var sourceFs1 = makeLinkHostFS(sourceCas1, 0, 0, makeLinkFS(sourceCas1, role, 0, 0)); setFeature(sourceFs1, "f1", "foo"); - var sourceFs2 = makeLinkHostFS(sourceCas, 0, 0, makeLinkFS(sourceCas, role, 1, 1)); + var sourceFs2 = makeLinkHostFS(sourceCas1, 0, 0, makeLinkFS(sourceCas1, role, 1, 1)); setFeature(sourceFs2, "f1", "bar"); // Set up target CAS @@ -190,9 +192,9 @@ public void thatSecondLinkWithSameTargetIsRejectedWhenRolesAreDisabled() throws slotFeature.setTraits(toJsonString(traits)); // Set up source CAS - var sourceFs = buildAnnotation(sourceCas.getCas(), HOST_TYPE) // + var sourceFs = buildAnnotation(sourceCas1.getCas(), HOST_TYPE) // .at(0, 0) // - .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, null, 0, 0))) + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas1, null, 0, 0))) .buildAndAddToIndexes(); // Set up target CAS @@ -220,9 +222,9 @@ public void thatSecondLinkWithSameTargetButDifferentRoleIsAddedWhenRolesAreEnabl throws Exception { // Set up source CAS - var sourceFs = buildAnnotation(sourceCas.getCas(), HOST_TYPE) // + var sourceFs = buildAnnotation(sourceCas1.getCas(), HOST_TYPE) // .at(0, 0) // - .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role1", 0, 0))) + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas1, "role1", 0, 0))) .buildAndAddToIndexes(); // Set up target CAS @@ -251,9 +253,9 @@ public void thatSecondLinkWithSameTargetAndSameRoleIsRejectedWhenRolesAreEnabled throws Exception { // Set up source CAS - var sourceFs = buildAnnotation(sourceCas.getCas(), HOST_TYPE) // + var sourceFs = buildAnnotation(sourceCas1.getCas(), HOST_TYPE) // .at(0, 0) // - .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role1", 0, 0))) + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas1, "role1", 0, 0))) .buildAndAddToIndexes(); // Set up target CAS @@ -278,7 +280,7 @@ public void thatSecondLinkWithSameTargetAndSameRoleIsRejectedWhenRolesAreEnabled } @Nested - class ThesholdBasedMergeStrategyTests + class SingleUserThesholdBasedMergeStrategyTests { @BeforeEach void setup() throws Exception @@ -293,24 +295,24 @@ void setup() throws Exception sut.setMergeStrategy(ThresholdBasedMergeStrategy.builder() // .withUserThreshold(1) // .withConfidenceThreshold(0) // - .withTopRanks(Integer.MAX_VALUE) // + .withTopRanks(0) // .build()); } @Test public void thatStackedLinkHostsWithDifferentTargetsAreMerged() throws Exception { - buildAnnotation(sourceCas.getCas(), HOST_TYPE) // + buildAnnotation(sourceCas1.getCas(), HOST_TYPE) // .at(0, 0) // - .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role1", 1, 1))) + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas1, "role1", 1, 1))) .buildAndAddToIndexes(); - buildAnnotation(sourceCas.getCas(), HOST_TYPE) // + buildAnnotation(sourceCas1.getCas(), HOST_TYPE) // .at(0, 0) // - .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas, "role2", 1, 1))) + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas1, "role2", 1, 1))) .buildAndAddToIndexes(); - var casMap = Map.of("source", sourceCas.getCas()); + var casMap = Map.of("source", sourceCas1.getCas()); var diff = doDiff(diffAdapters, casMap).toResult(); sut.mergeCas(diff, document, DUMMY_USER, targetCas.getCas(), casMap); @@ -326,4 +328,57 @@ public void thatStackedLinkHostsWithDifferentTargetsAreMerged() throws Exception 1))); } } + + @Nested + class DualUserThesholdBasedMergeStrategyTests + { + @BeforeEach + void setup() throws Exception + { + slotLayer.setOverlapMode(ANY_OVERLAP); + var traits = new LinkFeatureTraits(); + traits.setDiffMode(INCLUDE); + slotHostDiffAdapter.addLinkFeature("links", "role", "target", ONE_TARGET_MULTIPLE_ROLES, + INCLUDE); + + slotFeature.setTraits(toJsonString(traits)); + sut.setMergeStrategy(ThresholdBasedMergeStrategy.builder() // + .withUserThreshold(2) // + .withConfidenceThreshold(0) // + .withTopRanks(0) // + .build()); + } + + @Test + public void thatMatchingStackedLinksAreMerged() throws Exception + { + buildAnnotation(sourceCas1.getCas(), HOST_TYPE) // + .at(0, 0) // + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas1, "role1", 1, 1))) + .buildAndAddToIndexes(); + + buildAnnotation(sourceCas1.getCas(), HOST_TYPE) // + .at(0, 0) // + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas1, "role2", 1, 1))) + .buildAndAddToIndexes(); + + buildAnnotation(sourceCas2.getCas(), HOST_TYPE) // + .at(0, 0) // + .withFeature(LINKS_FEATURE, asList(makeLinkFS(sourceCas2, "role1", 1, 1))) + .buildAndAddToIndexes(); + + var casMap = Map.of("source1", sourceCas1.getCas(), "source2", sourceCas2.getCas()); + var diff = doDiff(diffAdapters, casMap).toResult(); + sut.mergeCas(diff, document, DUMMY_USER, targetCas.getCas(), casMap); + + var targetHosts = targetCas.select(HOST_TYPE).asList(); + assertThat(targetHosts) // + .as("Links by host in target CAS") // + .hasSize(1) // + .extracting(host -> toMaterializedLinks(host, LINKS_FEATURE, "role", "target")) // + .containsExactlyInAnyOrder( // + asList(new MaterializedLink(LINKS_FEATURE, "role1", Token._TypeName, 1, + 1))); + } + } } From 9adc530587032d927d6809167fc0be1321f87a48 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 3 Nov 2024 18:28:18 +0100 Subject: [PATCH 034/106] #5127 - Enable score on curation suggestions by default - Enable by default --- .../ui/curation/sidebar/CurationSidebarServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebarServiceImpl.java b/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebarServiceImpl.java index 31a9c6836d7..3cefc5b2d58 100644 --- a/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebarServiceImpl.java +++ b/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebarServiceImpl.java @@ -581,7 +581,7 @@ private class CurationSession // the curationdoc can be retrieved from user (CURATION or current) and projectId private String curationTarget; private boolean showAll; - private boolean showScore; + private boolean showScore = true; public CurationSession(String aUser) { From 9a9b168caddce6676eb718831b1d87001b7b58ef Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 4 Nov 2024 08:53:53 +0100 Subject: [PATCH 035/106] #5129 - Dynamic workflow may kick in for curator when try to view document from non-curator - Use the session owner instead of the data owner when checking for curator role --- .../DynamicWorkflowActionBarExtension.java | 20 +++++++++++-------- ...namicWorkloadManagerAutoConfiguration.java | 6 ++++-- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/annotation/DynamicWorkflowActionBarExtension.java b/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/annotation/DynamicWorkflowActionBarExtension.java index f884dbc612b..2866a2e97a0 100644 --- a/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/annotation/DynamicWorkflowActionBarExtension.java +++ b/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/annotation/DynamicWorkflowActionBarExtension.java @@ -25,6 +25,7 @@ import de.tudarmstadt.ukp.clarin.webanno.api.annotation.actionbar.ActionBarExtension; import de.tudarmstadt.ukp.clarin.webanno.api.annotation.page.AnnotationPageBase; +import de.tudarmstadt.ukp.clarin.webanno.security.UserDao; import de.tudarmstadt.ukp.inception.project.api.ProjectService; import de.tudarmstadt.ukp.inception.workload.dynamic.config.DynamicWorkloadManagerAutoConfiguration; import de.tudarmstadt.ukp.inception.workload.extension.WorkloadManagerExtension; @@ -41,13 +42,15 @@ public class DynamicWorkflowActionBarExtension { private final WorkloadManagementService workloadManagementService; private final ProjectService projectService; + private final UserDao userService; @Autowired public DynamicWorkflowActionBarExtension(WorkloadManagementService aWorkloadManagementService, - ProjectService aProjectService) + ProjectService aProjectService, UserDao aUserService) { workloadManagementService = aWorkloadManagementService; projectService = aProjectService; + userService = aUserService; } @Override @@ -65,17 +68,18 @@ public int getPriority() @Override public boolean accepts(AnnotationPageBase aPage) { - // #Issue 1813 fix - if (aPage.getModelObject().getProject() == null) { + // Issue #1813 fix + var project = aPage.getModelObject().getProject(); + if (project == null) { return false; } // Curator are excluded from the feature - return DYNAMIC_WORKLOAD_MANAGER_EXTENSION_ID - .equals(workloadManagementService.loadOrCreateWorkloadManagerConfiguration( - aPage.getModelObject().getProject()).getType()) - && !projectService.hasRole(aPage.getModelObject().getUser(), - aPage.getModelObject().getProject(), CURATOR); + var sessionOwner = userService.getCurrentUser(); + var workloadConfig = workloadManagementService + .loadOrCreateWorkloadManagerConfiguration(project); + return DYNAMIC_WORKLOAD_MANAGER_EXTENSION_ID.equals(workloadConfig.getType()) + && !projectService.hasRole(sessionOwner, project, CURATOR); } @Override diff --git a/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/config/DynamicWorkloadManagerAutoConfiguration.java b/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/config/DynamicWorkloadManagerAutoConfiguration.java index 2784e91c355..f8f4c96c77b 100644 --- a/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/config/DynamicWorkloadManagerAutoConfiguration.java +++ b/inception/inception-workload-dynamic/src/main/java/de/tudarmstadt/ukp/inception/workload/dynamic/config/DynamicWorkloadManagerAutoConfiguration.java @@ -84,9 +84,11 @@ public DynamicWorkloadStateWatcher dynamicWorkloadStateWatcher( @Bean public DynamicWorkflowActionBarExtension dynamicWorkflowActionBarExtension( - WorkloadManagementService aWorkloadManagementService, ProjectService aProjectService) + WorkloadManagementService aWorkloadManagementService, ProjectService aProjectService, + UserDao aUserService) { - return new DynamicWorkflowActionBarExtension(aWorkloadManagementService, aProjectService); + return new DynamicWorkflowActionBarExtension(aWorkloadManagementService, aProjectService, + aUserService); } @Bean From cc242c2fbe5731a8eba2e50cd5b37f13a14d78f4 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 4 Nov 2024 08:57:28 +0100 Subject: [PATCH 036/106] #5130 - Searching Huggingface hub should only return endpoint-compatible models - Search only for models compatibe with the endpoints service --- .../imls/hf/client/HfHubClientImpl.java | 1 + .../imls/hf/model/HfModelCard.java | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java b/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java index 734892ac2bf..408a530f62e 100644 --- a/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java +++ b/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java @@ -64,6 +64,7 @@ public HfModelCard[] listModels(String aSearch) throws IOException Map queryParameters = new LinkedHashMap<>(); queryParameters.put("cardData", "true"); queryParameters.put("pipeline_tag", "token-classification"); + queryParameters.put("other", "endpoints_compatible"); if (isNotBlank(aSearch)) { queryParameters.put("search", aSearch); } diff --git a/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/model/HfModelCard.java b/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/model/HfModelCard.java index d116d88c32b..ad617e7382d 100644 --- a/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/model/HfModelCard.java +++ b/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/model/HfModelCard.java @@ -18,6 +18,7 @@ package de.tudarmstadt.ukp.inception.recommendation.imls.hf.model; import java.io.Serializable; +import java.util.Set; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -34,6 +35,10 @@ public class HfModelCard private @JsonProperty("pipeline_tag") String pipelineTag; // private @JsonProperty("cardData") String pipelineTag; private @JsonProperty("modelId") String modelId; + private @JsonProperty("tags") Set tags; + // private @JsonProperty("language") String langauge; + // private @JsonProperty("license") String license; + private @JsonProperty("library_name") String libraryName; public boolean isPrivateAccess() { @@ -64,4 +69,24 @@ public void setModelId(String aModelId) { modelId = aModelId; } + + public Set getTags() + { + return tags; + } + + public void setTags(Set aTags) + { + tags = aTags; + } + + public String getLibraryName() + { + return libraryName; + } + + public void setLibraryName(String aLibraryName) + { + libraryName = aLibraryName; + } } From c56be7176cbde8cf2a1591d0cc50a196614a0c09 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 4 Nov 2024 09:10:49 +0100 Subject: [PATCH 037/106] #5133 - Add keybindings to named entity template sample data - Added keybindings --- .../EntityAnnotationProjectInitializer.java | 36 ++++++++++++++++--- ...gProjectInitializersAutoConfiguration.java | 6 ++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java index c05bd48d09a..b162fb8994b 100644 --- a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java +++ b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java @@ -17,6 +17,8 @@ */ package de.tudarmstadt.ukp.inception.project.initializers.neannotation; +import static java.util.Arrays.asList; + import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -29,6 +31,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.core.annotation.Order; +import de.tudarmstadt.ukp.clarin.webanno.api.annotation.keybindings.KeyBinding; import de.tudarmstadt.ukp.clarin.webanno.model.Project; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; import de.tudarmstadt.ukp.clarin.webanno.project.initializers.NamedEntityLayerInitializer; @@ -36,6 +39,9 @@ import de.tudarmstadt.ukp.clarin.webanno.security.UserDao; import de.tudarmstadt.ukp.clarin.webanno.text.LineOrientedTextFormatSupport; import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; +import de.tudarmstadt.ukp.inception.annotation.feature.string.StringFeatureSupport; +import de.tudarmstadt.ukp.inception.annotation.feature.string.StringFeatureTraits; +import de.tudarmstadt.ukp.inception.annotation.feature.string.StringFeatureTraits.EditorType; import de.tudarmstadt.ukp.inception.documents.api.DocumentService; import de.tudarmstadt.ukp.inception.io.jsoncas.UimaJsonCasFormatSupport; import de.tudarmstadt.ukp.inception.project.api.ProjectInitializationRequest; @@ -61,15 +67,17 @@ public class EntityAnnotationProjectInitializer private final ApplicationContext context; private final DocumentService documentService; private final UserDao userService; + private final StringFeatureSupport stringFeatureSupport; public EntityAnnotationProjectInitializer(ApplicationContext aContext, AnnotationSchemaService aAnnotationService, DocumentService aDocumentService, - UserDao aUserService) + UserDao aUserService, StringFeatureSupport aStringFeatureSupport) { context = aContext; annotationService = aAnnotationService; documentService = aDocumentService; userService = aUserService; + stringFeatureSupport = aStringFeatureSupport; } @Override @@ -132,17 +140,35 @@ public void configure(ProjectInitializationRequest aRequest) throws IOException var project = aRequest.getProject(); project.setName(userService.getCurrentUsername() + " - New entity annotation project"); - var tagset = annotationService - .getTagSet(NamedEntitySampleDataTagSetInitializer.TAG_SET_NAME, project); - var layer = annotationService.findLayer(project, NamedEntity.class.getName()); + var identifierFeature = annotationService.getFeature(NamedEntity._FeatName_identifier, layer); identifierFeature.setEnabled(false); annotationService.createFeature(identifierFeature); + var valueFeature = annotationService.getFeature(NamedEntity._FeatName_value, layer); valueFeature.setEnabled(true); - valueFeature.setTagset(tagset); + + if (aRequest.isIncludeSampleData()) { + var tagset = annotationService + .getTagSet(NamedEntitySampleDataTagSetInitializer.TAG_SET_NAME, project); + valueFeature.setTagset(tagset); + + var valueFeatureTraits = new StringFeatureTraits(); + valueFeatureTraits.setEditorType(EditorType.RADIOGROUP); + valueFeatureTraits.setKeyBindings(asList( // + new KeyBinding("1", "Date"), // + new KeyBinding("2", "Event"), // + new KeyBinding("3", "Location"), // + new KeyBinding("4", "Organization"), // + new KeyBinding("5", "Person"), // + new KeyBinding("6", "Product"), // + new KeyBinding("7", "Product Category"), // + new KeyBinding("8", "Route"))); + stringFeatureSupport.writeTraits(valueFeature, valueFeatureTraits); + } + annotationService.createFeature(valueFeature); var description = // diff --git a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java index bc05e09b059..ae4d2749a25 100644 --- a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java +++ b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java @@ -24,6 +24,7 @@ import org.springframework.context.annotation.Configuration; import de.tudarmstadt.ukp.clarin.webanno.security.UserDao; +import de.tudarmstadt.ukp.inception.annotation.feature.string.StringFeatureSupport; import de.tudarmstadt.ukp.inception.documents.api.DocumentService; import de.tudarmstadt.ukp.inception.kb.KnowledgeBaseService; import de.tudarmstadt.ukp.inception.kb.config.KnowledgeBaseProperties; @@ -78,10 +79,11 @@ public EntityLinkingProjectInitializer entityLinkingProjectInitializer( @Bean public EntityAnnotationProjectInitializer entityAnnotationProjectInitializer( ApplicationContext aContext, AnnotationSchemaService aAnnotationService, - DocumentService aDocumentService, UserDao aUserService) + DocumentService aDocumentService, UserDao aUserService, + StringFeatureSupport aStringFeatureSupport) { return new EntityAnnotationProjectInitializer(aContext, aAnnotationService, - aDocumentService, aUserService); + aDocumentService, aUserService, aStringFeatureSupport); } @ConditionalOnBean(RecommendationService.class) From 35e7df6835d0f28000949403d98fe0ac63c66cc0 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 4 Nov 2024 18:55:48 +0100 Subject: [PATCH 038/106] #5133 - Add keybindings to named entity template sample data - Fix dependencies --- .../inception-project-initializers-wikidatalinking/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/inception/inception-project-initializers-wikidatalinking/pom.xml b/inception/inception-project-initializers-wikidatalinking/pom.xml index 05a2ee716d0..359be93b949 100644 --- a/inception/inception-project-initializers-wikidatalinking/pom.xml +++ b/inception/inception-project-initializers-wikidatalinking/pom.xml @@ -105,6 +105,11 @@ inception-io-text ${project.version} + + de.tudarmstadt.ukp.inception.app + inception-api-annotation + ${project.version} + org.apache.uima From 91cffb41c687dafd41c682249e7c528fa314201f Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 4 Nov 2024 19:17:31 +0100 Subject: [PATCH 039/106] #5135 - Cannot jump to curation suggestion without accepting it - Change the scrollTo JS API function to accept multiple offsets (although only the first one is currently ever used) - Fix ScrollToHandler to jump without taking the window offset into consideration - Fix the annotation sidebar to properly jump --- .../main/ts/src/AnnotationsByLabelList.svelte | 5 ++ .../main/ts/src/AnnotationsByLayerList.svelte | 12 ++-- .../actions/EditorAjaxRequestHandler.java | 1 + .../diam/editor/actions/ScrollToHandler.java | 66 ++++++++++++++----- .../src/main/ts/src/diam/DiamAjaxImpl.ts | 12 +++- .../src/main/ts/src/diam/DiamAjax.ts | 2 +- 6 files changed, 71 insertions(+), 27 deletions(-) diff --git a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte index fffece6ffd6..eafdeb7649e 100644 --- a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte +++ b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte @@ -100,6 +100,11 @@ } function scrollTo(ann: Annotation) { + if (ann instanceof Span) { + ajaxClient.scrollTo({ id: ann.vid, offset: ann.offsets[0] }); + return; + } + ajaxClient.scrollTo({ id: ann.vid }); } diff --git a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte index 8e93d6c2ae5..94682be88e6 100644 --- a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte +++ b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte @@ -102,8 +102,8 @@ } } - function scrollTo(ann: Annotation) { - ajaxClient.scrollTo({ id: ann.vid }); + function scrollToSpan(span: Span) { + ajaxClient.scrollTo({ id: span.vid, offset: span.offsets[0] }); } function mouseOverAnnotation(event: MouseEvent, annotation: Annotation) { @@ -218,7 +218,7 @@ > -
scrollTo(ann)}> +
scrollToSpan(ann)}>
scrollTo(target)} + on:click={() => scrollToSpan(target)} >
-
scrollTo(ann.arguments[0].target)}> +
scrollToSpan(ann.arguments[0].target)}>
↳
@@ -290,7 +290,7 @@ span={ann.arguments[0].target} />
-
scrollTo(ann.arguments[1].target)}> +
scrollToSpan(ann.arguments[1].target)}>
↳
diff --git a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java index 3822ddfdebe..84d443915e3 100644 --- a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java +++ b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java @@ -30,6 +30,7 @@ public interface EditorAjaxRequestHandler { int PRIO_CONTEXT_MENU = -10; int PRIO_RENDER_HANDLER = 0; + int PRIO_NAVIGATE_HANDLER = 50; int PRIO_SLOT_FILLER_HANDLER = 100; int PRIO_UNARM_SLOT_HANDLER = 180; int PRIO_EXTENSION_HANDLER = 190; diff --git a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java index b6eac6cfd9f..122b5dfb236 100644 --- a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java +++ b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java @@ -17,7 +17,9 @@ */ package de.tudarmstadt.ukp.inception.diam.editor.actions; -import static de.tudarmstadt.ukp.inception.diam.editor.actions.CreateSpanAnnotationHandler.getRangeFromRequest; +import static de.tudarmstadt.ukp.inception.rendering.model.Range.rangeClippedToDocument; + +import java.io.IOException; import org.apache.uima.cas.CAS; import org.apache.wicket.ajax.AjaxRequestTarget; @@ -29,15 +31,18 @@ import de.tudarmstadt.ukp.inception.diam.editor.DiamAjaxBehavior; import de.tudarmstadt.ukp.inception.diam.editor.config.DiamAutoConfig; import de.tudarmstadt.ukp.inception.diam.model.ajax.DefaultAjaxResponse; +import de.tudarmstadt.ukp.inception.diam.model.compact.CompactRangeList; import de.tudarmstadt.ukp.inception.rendering.model.Range; import de.tudarmstadt.ukp.inception.rendering.vmodel.VID; +import de.tudarmstadt.ukp.inception.schema.api.adapter.AnnotationException; +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; /** *

* This class is exposed as a Spring Component via {@link DiamAutoConfig#scrollToHandler}. *

*/ -@Order(EditorAjaxRequestHandler.PRIO_ANNOTATION_HANDLER) +@Order(EditorAjaxRequestHandler.PRIO_NAVIGATE_HANDLER) public class ScrollToHandler extends EditorAjaxRequestHandlerBase { @@ -54,24 +59,12 @@ public DefaultAjaxResponse handle(DiamAjaxBehavior aBehavior, AjaxRequestTarget Request aRequest) { try { - AnnotationPageBase page = getPage(); - CAS cas = page.getEditorCas(); - - IRequestParameters requestParameters = aRequest.getRequestParameters(); + var page = getPage(); + var cas = page.getEditorCas(); - if (!requestParameters.getParameterValue(PARAM_OFFSETS).isEmpty()) { - Range offsets = getRangeFromRequest(getAnnotatorState(), requestParameters, cas); - page.getAnnotationActionHandler().actionJump(aTarget, offsets.getBegin(), - offsets.getEnd()); - } - else { - VID vid = VID.parseOptional( - requestParameters.getParameterValue(PARAM_ID).toOptionalString()); + var requestParameters = aRequest.getRequestParameters(); - if (vid.isSet() && !vid.isSynthetic()) { - page.getAnnotationActionHandler().actionJump(aTarget, vid); - } - } + scrollTo(aTarget, page, cas, requestParameters); return new DefaultAjaxResponse(getAction(aRequest)); } @@ -79,4 +72,41 @@ public DefaultAjaxResponse handle(DiamAjaxBehavior aBehavior, AjaxRequestTarget return handleError("Unable to scroll to annotation", e); } } + + private void scrollTo(AjaxRequestTarget aTarget, AnnotationPageBase page, CAS cas, + IRequestParameters requestParameters) + throws IOException, AnnotationException + { + var vid = VID + .parseOptional(requestParameters.getParameterValue(PARAM_ID).toOptionalString()); + + if (vid.isSet() && !vid.isSynthetic()) { + page.getAnnotationActionHandler().actionJump(aTarget, vid); + return; + } + + if (!requestParameters.getParameterValue(PARAM_OFFSETS).isEmpty()) { + var offsets = getRangeFromRequest(requestParameters, cas); + page.getAnnotationActionHandler().actionJump(aTarget, offsets.getBegin(), + offsets.getEnd()); + return; + } + } + + /** + * Extract offset information from the current request. These are either offsets of an existing + * selected annotations or offsets contained in the request for the creation of a new + * annotation. + */ + private Range getRangeFromRequest(IRequestParameters request, CAS aCas) throws IOException + { + var offsets = request.getParameterValue(PARAM_OFFSETS).toString(); + + var offsetLists = JSONUtil.fromJsonString(CompactRangeList.class, offsets); + + var begin = offsetLists.get(0).getBegin(); + var end = offsetLists.get(offsetLists.size() - 1).getEnd(); + + return rangeClippedToDocument(aCas, begin, end); + } } diff --git a/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts b/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts index 9d9d6b3765f..a485735a5cc 100644 --- a/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts +++ b/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts @@ -76,7 +76,15 @@ export class DiamAjaxImpl implements DiamAjax { }) } - scrollTo (args: { id?: VID, offset?: Offsets }): void { + scrollTo (args: { id?: VID, offset?: Offsets, offsets?: Array }): void { + let effectiveOffsets: Array | undefined + if (args.offset) { + effectiveOffsets = [args.offset] + } + else { + effectiveOffsets = args.offsets + } + DiamAjaxImpl.performAjaxCall(() => { Wicket.Ajax.ajax({ m: 'POST', @@ -84,7 +92,7 @@ export class DiamAjaxImpl implements DiamAjax { ep: { action: 'scrollTo', id: args.id, - offset: args.offset + offsets: JSON.stringify(effectiveOffsets) } }) }) diff --git a/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts b/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts index e29c2d5ab7c..348f87cb665 100644 --- a/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts +++ b/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts @@ -45,7 +45,7 @@ export interface DiamAjax { * Scroll to the given annotation or offset. If both are given, the offset is used. Offsets * are relative to the entire document, not to the current viewport. */ - scrollTo(args: { id?: VID, offset?: Offsets }): void; + scrollTo(args: { id?: VID, offset?: Offsets, offsets?: Array }): void; /** * Delete the given annotation. From 5fbe17bf107caede8635ca2480346ec8ae43e3bc Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Mon, 4 Nov 2024 19:17:31 +0100 Subject: [PATCH 040/106] #5135 - Cannot jump to curation suggestion without accepting it - Change the scrollTo JS API function to accept multiple offsets (although only the first one is currently ever used) - Fix ScrollToHandler to jump without taking the window offset into consideration - Fix the annotation sidebar to properly jump --- .../main/ts/src/AnnotationsByLabelList.svelte | 5 ++ .../main/ts/src/AnnotationsByLayerList.svelte | 12 ++-- .../ts/src/AnnotationsByPositionList.svelte | 2 +- .../actions/EditorAjaxRequestHandler.java | 1 + .../diam/editor/actions/ScrollToHandler.java | 66 ++++++++++++++----- .../src/main/ts/src/diam/DiamAjaxImpl.ts | 12 +++- .../src/main/ts/src/diam/DiamAjax.ts | 2 +- 7 files changed, 72 insertions(+), 28 deletions(-) diff --git a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte index fffece6ffd6..eafdeb7649e 100644 --- a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte +++ b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLabelList.svelte @@ -100,6 +100,11 @@ } function scrollTo(ann: Annotation) { + if (ann instanceof Span) { + ajaxClient.scrollTo({ id: ann.vid, offset: ann.offsets[0] }); + return; + } + ajaxClient.scrollTo({ id: ann.vid }); } diff --git a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte index 8e93d6c2ae5..94682be88e6 100644 --- a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte +++ b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByLayerList.svelte @@ -102,8 +102,8 @@ } } - function scrollTo(ann: Annotation) { - ajaxClient.scrollTo({ id: ann.vid }); + function scrollToSpan(span: Span) { + ajaxClient.scrollTo({ id: span.vid, offset: span.offsets[0] }); } function mouseOverAnnotation(event: MouseEvent, annotation: Annotation) { @@ -218,7 +218,7 @@ > -
scrollTo(ann)}> +
scrollToSpan(ann)}>
scrollTo(target)} + on:click={() => scrollToSpan(target)} >
-
scrollTo(ann.arguments[0].target)}> +
scrollToSpan(ann.arguments[0].target)}>
↳
@@ -290,7 +290,7 @@ span={ann.arguments[0].target} />
-
scrollTo(ann.arguments[1].target)}> +
scrollToSpan(ann.arguments[1].target)}>
↳
diff --git a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByPositionList.svelte b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByPositionList.svelte index 523f45cc518..48cd0897c4c 100644 --- a/inception/inception-diam-editor/src/main/ts/src/AnnotationsByPositionList.svelte +++ b/inception/inception-diam-editor/src/main/ts/src/AnnotationsByPositionList.svelte @@ -59,7 +59,7 @@ } function scrollToRelation(relation: Relation) { - ajaxClient.scrollTo({ id: relation.vid }); + ajaxClient.scrollTo({ id: relation.vid, offset: relation.arguments[0].target.offsets[0] }); } function mouseOverAnnotation(event: MouseEvent, annotation: Annotation) { diff --git a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java index 3822ddfdebe..84d443915e3 100644 --- a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java +++ b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/EditorAjaxRequestHandler.java @@ -30,6 +30,7 @@ public interface EditorAjaxRequestHandler { int PRIO_CONTEXT_MENU = -10; int PRIO_RENDER_HANDLER = 0; + int PRIO_NAVIGATE_HANDLER = 50; int PRIO_SLOT_FILLER_HANDLER = 100; int PRIO_UNARM_SLOT_HANDLER = 180; int PRIO_EXTENSION_HANDLER = 190; diff --git a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java index b6eac6cfd9f..122b5dfb236 100644 --- a/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java +++ b/inception/inception-diam/src/main/java/de/tudarmstadt/ukp/inception/diam/editor/actions/ScrollToHandler.java @@ -17,7 +17,9 @@ */ package de.tudarmstadt.ukp.inception.diam.editor.actions; -import static de.tudarmstadt.ukp.inception.diam.editor.actions.CreateSpanAnnotationHandler.getRangeFromRequest; +import static de.tudarmstadt.ukp.inception.rendering.model.Range.rangeClippedToDocument; + +import java.io.IOException; import org.apache.uima.cas.CAS; import org.apache.wicket.ajax.AjaxRequestTarget; @@ -29,15 +31,18 @@ import de.tudarmstadt.ukp.inception.diam.editor.DiamAjaxBehavior; import de.tudarmstadt.ukp.inception.diam.editor.config.DiamAutoConfig; import de.tudarmstadt.ukp.inception.diam.model.ajax.DefaultAjaxResponse; +import de.tudarmstadt.ukp.inception.diam.model.compact.CompactRangeList; import de.tudarmstadt.ukp.inception.rendering.model.Range; import de.tudarmstadt.ukp.inception.rendering.vmodel.VID; +import de.tudarmstadt.ukp.inception.schema.api.adapter.AnnotationException; +import de.tudarmstadt.ukp.inception.support.json.JSONUtil; /** *

* This class is exposed as a Spring Component via {@link DiamAutoConfig#scrollToHandler}. *

*/ -@Order(EditorAjaxRequestHandler.PRIO_ANNOTATION_HANDLER) +@Order(EditorAjaxRequestHandler.PRIO_NAVIGATE_HANDLER) public class ScrollToHandler extends EditorAjaxRequestHandlerBase { @@ -54,24 +59,12 @@ public DefaultAjaxResponse handle(DiamAjaxBehavior aBehavior, AjaxRequestTarget Request aRequest) { try { - AnnotationPageBase page = getPage(); - CAS cas = page.getEditorCas(); - - IRequestParameters requestParameters = aRequest.getRequestParameters(); + var page = getPage(); + var cas = page.getEditorCas(); - if (!requestParameters.getParameterValue(PARAM_OFFSETS).isEmpty()) { - Range offsets = getRangeFromRequest(getAnnotatorState(), requestParameters, cas); - page.getAnnotationActionHandler().actionJump(aTarget, offsets.getBegin(), - offsets.getEnd()); - } - else { - VID vid = VID.parseOptional( - requestParameters.getParameterValue(PARAM_ID).toOptionalString()); + var requestParameters = aRequest.getRequestParameters(); - if (vid.isSet() && !vid.isSynthetic()) { - page.getAnnotationActionHandler().actionJump(aTarget, vid); - } - } + scrollTo(aTarget, page, cas, requestParameters); return new DefaultAjaxResponse(getAction(aRequest)); } @@ -79,4 +72,41 @@ public DefaultAjaxResponse handle(DiamAjaxBehavior aBehavior, AjaxRequestTarget return handleError("Unable to scroll to annotation", e); } } + + private void scrollTo(AjaxRequestTarget aTarget, AnnotationPageBase page, CAS cas, + IRequestParameters requestParameters) + throws IOException, AnnotationException + { + var vid = VID + .parseOptional(requestParameters.getParameterValue(PARAM_ID).toOptionalString()); + + if (vid.isSet() && !vid.isSynthetic()) { + page.getAnnotationActionHandler().actionJump(aTarget, vid); + return; + } + + if (!requestParameters.getParameterValue(PARAM_OFFSETS).isEmpty()) { + var offsets = getRangeFromRequest(requestParameters, cas); + page.getAnnotationActionHandler().actionJump(aTarget, offsets.getBegin(), + offsets.getEnd()); + return; + } + } + + /** + * Extract offset information from the current request. These are either offsets of an existing + * selected annotations or offsets contained in the request for the creation of a new + * annotation. + */ + private Range getRangeFromRequest(IRequestParameters request, CAS aCas) throws IOException + { + var offsets = request.getParameterValue(PARAM_OFFSETS).toString(); + + var offsetLists = JSONUtil.fromJsonString(CompactRangeList.class, offsets); + + var begin = offsetLists.get(0).getBegin(); + var end = offsetLists.get(offsetLists.size() - 1).getEnd(); + + return rangeClippedToDocument(aCas, begin, end); + } } diff --git a/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts b/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts index 9d9d6b3765f..a485735a5cc 100644 --- a/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts +++ b/inception/inception-diam/src/main/ts/src/diam/DiamAjaxImpl.ts @@ -76,7 +76,15 @@ export class DiamAjaxImpl implements DiamAjax { }) } - scrollTo (args: { id?: VID, offset?: Offsets }): void { + scrollTo (args: { id?: VID, offset?: Offsets, offsets?: Array }): void { + let effectiveOffsets: Array | undefined + if (args.offset) { + effectiveOffsets = [args.offset] + } + else { + effectiveOffsets = args.offsets + } + DiamAjaxImpl.performAjaxCall(() => { Wicket.Ajax.ajax({ m: 'POST', @@ -84,7 +92,7 @@ export class DiamAjaxImpl implements DiamAjax { ep: { action: 'scrollTo', id: args.id, - offset: args.offset + offsets: JSON.stringify(effectiveOffsets) } }) }) diff --git a/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts b/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts index e29c2d5ab7c..348f87cb665 100644 --- a/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts +++ b/inception/inception-js-api/src/main/ts/src/diam/DiamAjax.ts @@ -45,7 +45,7 @@ export interface DiamAjax { * Scroll to the given annotation or offset. If both are given, the offset is used. Offsets * are relative to the entire document, not to the current viewport. */ - scrollTo(args: { id?: VID, offset?: Offsets }): void; + scrollTo(args: { id?: VID, offset?: Offsets, offsets?: Array }): void; /** * Delete the given annotation. From e0250ad4fdd247c0a9f88ff4b54b63937f7a96bc Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 5 Nov 2024 11:59:28 +0100 Subject: [PATCH 041/106] #5127 - Enable score on curation suggestions by default - Enable by default --- .../ukp/inception/ui/curation/sidebar/CurationSidebar.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebar.java b/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebar.java index e54996636cd..8d8b77b9f01 100644 --- a/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebar.java +++ b/inception/inception-ui-curation/src/main/java/de/tudarmstadt/ukp/inception/ui/curation/sidebar/CurationSidebar.java @@ -133,13 +133,13 @@ public CurationSidebar(String aId, AnnotationActionHandler aActionHandler, queue(usersForm = createUserSelection("usersForm")); - showMerged = new CheckBox("showMerged", Model.of()); + showMerged = new CheckBox("showMerged", Model.of(false)); showMerged.add(visibleWhen(this::isSessionActive)); showMerged.add(new LambdaAjaxFormComponentUpdatingBehavior(CHANGE_EVENT, this::actionToggleShowMerged)); queue(showMerged); - showScore = new CheckBox("showScore", Model.of()); + showScore = new CheckBox("showScore", Model.of(true)); showScore.add(visibleWhen(this::isSessionActive)); showScore.add(new LambdaAjaxFormComponentUpdatingBehavior(CHANGE_EVENT, this::actionToggleShowScore)); From a9b833eb00c54766d88b7ac75ef65fa029858a9e Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 5 Nov 2024 12:40:39 +0100 Subject: [PATCH 042/106] No issue: Try re-enabling the dependency submission and attach build artifacts --- .github/workflows/maven.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a2a30c00fc6..d540d6a0f5a 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -52,7 +52,11 @@ jobs: - name: Build with Maven run: mvn --no-transfer-progress -B clean verify --file pom.xml - # Fails with error message - no idea why... - # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive - # - name: Update dependency graph - # uses: advanced-security/maven-dependency-submission-action@571e99aab1055c2e71a1e2309b9691de18d6b7d6 + - name: Update dependency graph + uses: advanced-security/maven-dependency-submission-action@v4 + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: inception/inception-app-webapp/target/inception-app-webapp-*-standalone.jar From 9ca1df78b3e473978fbedc682a782245206826d4 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 5 Nov 2024 13:12:50 +0100 Subject: [PATCH 043/106] No issue: Run dependency analysis only on push to avoid making the PRs build longer --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index d540d6a0f5a..c685961dbc6 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -53,6 +53,7 @@ jobs: run: mvn --no-transfer-progress -B clean verify --file pom.xml - name: Update dependency graph + if: github.event_name == 'push' uses: advanced-security/maven-dependency-submission-action@v4 - name: Upload Artifacts From 7487192bfe80cf8ada1a81042d089935632aae98 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 5 Nov 2024 14:40:08 +0100 Subject: [PATCH 044/106] No issue: Remove dependency analysis - it just takes forever and does not seem to finish --- .github/workflows/maven.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index c685961dbc6..e7e2c006b79 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -52,10 +52,6 @@ jobs: - name: Build with Maven run: mvn --no-transfer-progress -B clean verify --file pom.xml - - name: Update dependency graph - if: github.event_name == 'push' - uses: advanced-security/maven-dependency-submission-action@v4 - - name: Upload Artifacts uses: actions/upload-artifact@v4 with: From b1ae647b5eb9b6dce5ea875a37037d8803c0a995 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 5 Nov 2024 15:38:33 +0100 Subject: [PATCH 045/106] No issue: Upload only on the linux step --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index e7e2c006b79..8f1ca259671 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -53,6 +53,7 @@ jobs: run: mvn --no-transfer-progress -B clean verify --file pom.xml - name: Upload Artifacts + if: matrix.os == 'ubuntu-latest' uses: actions/upload-artifact@v4 with: name: build-artifacts From f45d19f3c770cae87d567c2739f45b9116109198 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Tue, 5 Nov 2024 20:03:29 +0100 Subject: [PATCH 046/106] No issue: Mention database schema changes in upgrade notes --- .../META-INF/asciidoc/admin-guide/upgrade_notes.adoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/inception/inception-doc/src/main/resources/META-INF/asciidoc/admin-guide/upgrade_notes.adoc b/inception/inception-doc/src/main/resources/META-INF/asciidoc/admin-guide/upgrade_notes.adoc index 5133fa73164..51098c403c1 100644 --- a/inception/inception-doc/src/main/resources/META-INF/asciidoc/admin-guide/upgrade_notes.adoc +++ b/inception/inception-doc/src/main/resources/META-INF/asciidoc/admin-guide/upgrade_notes.adoc @@ -22,6 +22,15 @@ NOTE: It is a good idea to back up your installation and data before an upgrade. you can actually re-create a working installation by restoring your backup. A backup which cannot be successfully restored is worthless. +== INCEpTION 34.0 + +=== Database schema changes + +This version makes various changes to the database schema in order to improve compatibiltiy with +different database systems. This entails that after running this version, you cannot easily downgrade +to an older version of {product-name} anymore. If you need to downgrade, you need to restore a +backup from before the upgrade or manually revert the database schema changes. + == INCEpTION 33.0 === MariaDB upgrade in the Docker Compose file From 1e2a477df67760995364d012f56f7e415ecbc78d Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 6 Nov 2024 20:17:28 +0100 Subject: [PATCH 047/106] #5141 - Interactive recommenders should not be listed in the bulk-processing - Skip interactive recommenders on the bulk processing panel --- .../processing/recommender/BulkRecommenderPanel.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkRecommenderPanel.java b/inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkRecommenderPanel.java index 1bae112500f..a7824dc35c2 100644 --- a/inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkRecommenderPanel.java +++ b/inception/inception-processing/src/main/java/de/tudarmstadt/ukp/inception/processing/recommender/BulkRecommenderPanel.java @@ -165,6 +165,10 @@ private List listRecommenders() var factory = maybeFactory.get(); var engine = factory.build(recommender); + if (factory.isInteractive(recommender)) { + continue; + } + // If a recommender requires training, it would yield no results if the user has not yet // annotated any documents. So in this case, we do currently not offer it for // processing. From 1a262a6dbd82c60c31b7e87ac3fbe286a78b6f82 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 6 Nov 2024 20:20:26 +0100 Subject: [PATCH 048/106] #5142 - DanglingRelationsCheck hammers database and runs very slow - Cache the adapter by type so we don't hammer the DB trying to find an adapter for each relation --- .../webanno/diag/checks/DanglingRelationsCheck.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java index eed28e1ece2..81b30dbebb2 100644 --- a/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java +++ b/inception/inception-diag/src/main/java/de/tudarmstadt/ukp/clarin/webanno/diag/checks/DanglingRelationsCheck.java @@ -21,10 +21,12 @@ import static de.tudarmstadt.ukp.inception.support.WebAnnoConst.FEAT_REL_TARGET; import static de.tudarmstadt.ukp.inception.support.logging.LogLevel.INFO; +import java.util.HashMap; import java.util.List; import org.apache.uima.cas.CAS; import org.apache.uima.cas.Feature; +import org.apache.uima.cas.Type; import org.apache.uima.cas.text.AnnotationFS; import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument; @@ -51,7 +53,9 @@ public DanglingRelationsCheck(AnnotationSchemaService aAnnotationService) public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, List aMessages) { - boolean ok = true; + var ok = true; + + var adapterCache = new HashMap(); for (var fs : aCas.getAnnotationIndex()) { var t = fs.getType(); @@ -64,8 +68,9 @@ public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, continue; } - var relationAdapter = (RelationAdapter) annotationService - .findAdapter(aDocument.getProject(), fs); + var relationAdapter = adapterCache.computeIfAbsent(t, + _t -> (RelationAdapter) annotationService.findAdapter(aDocument.getProject(), + fs)); Feature relationSourceAttachFeature = null; Feature relationTargetAttachFeature = null; @@ -88,7 +93,7 @@ public boolean check(SourceDocument aDocument, String aDataOwner, CAS aCas, target = (AnnotationFS) target.getFeatureValue(relationTargetAttachFeature); } - // Does it have null endpoints? + // Does it have null end-points? if (source == null || target == null) { var message = new StringBuilder(); From 6f6c358e063238c3a9c2a5dd310449a5b7215404 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 6 Nov 2024 20:30:08 +0100 Subject: [PATCH 049/106] #5145 - Managers and curators should be able to re-open their documents - Check if the user is a manager/curator on the current project before rejecting re-opening the document --- .../matrix/annotation/MatrixWorkflowActionBarItemGroup.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/inception/inception-workload-matrix/src/main/java/de/tudarmstadt/ukp/inception/workload/matrix/annotation/MatrixWorkflowActionBarItemGroup.java b/inception/inception-workload-matrix/src/main/java/de/tudarmstadt/ukp/inception/workload/matrix/annotation/MatrixWorkflowActionBarItemGroup.java index bb037cc48d4..891263fe3c4 100644 --- a/inception/inception-workload-matrix/src/main/java/de/tudarmstadt/ukp/inception/workload/matrix/annotation/MatrixWorkflowActionBarItemGroup.java +++ b/inception/inception-workload-matrix/src/main/java/de/tudarmstadt/ukp/inception/workload/matrix/annotation/MatrixWorkflowActionBarItemGroup.java @@ -21,6 +21,7 @@ import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState.IN_PROGRESS; import static de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentStateChangeFlag.EXPLICIT_ANNOTATOR_USER_ACTION; import static de.tudarmstadt.ukp.clarin.webanno.model.PermissionLevel.CURATOR; +import static de.tudarmstadt.ukp.clarin.webanno.model.PermissionLevel.MANAGER; import static de.tudarmstadt.ukp.clarin.webanno.model.SourceDocumentState.CURATION_FINISHED; import static de.tudarmstadt.ukp.clarin.webanno.model.SourceDocumentState.CURATION_IN_PROGRESS; import static de.tudarmstadt.ukp.inception.support.WebAnnoConst.CURATION_USER; @@ -280,8 +281,10 @@ private void actionToggleDocumentState(AjaxRequestTarget aTarget) return; } + var sessionOwner = userRepository.getCurrentUser(); var annDoc = documentService.getAnnotationDocument(document, state.getUser()); - if (annDoc.getAnnotatorState() != annDoc.getState()) { + if (annDoc.getAnnotatorState() != annDoc.getState() + && !projectService.hasRole(sessionOwner, state.getProject(), CURATOR, MANAGER)) { error("Annotation state has been overridden by a project manager or curator. " + "You cannot change it."); aTarget.addChildren(getPage(), IFeedback.class); From dc80914aa16d1cc2a294db9c8098be71de9ec15e Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 6 Nov 2024 20:43:50 +0100 Subject: [PATCH 050/106] #5147 - Relations should be visible in the annotation sidebar when render mode is never - Render relations in never mode when rendering for the annotation sidebar --- .../annotation/layer/relation/RelationRenderer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/layer/relation/RelationRenderer.java b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/layer/relation/RelationRenderer.java index ec962b18c58..38b6f33f9b4 100644 --- a/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/layer/relation/RelationRenderer.java +++ b/inception/inception-api-annotation/src/main/java/de/tudarmstadt/ukp/inception/annotation/layer/relation/RelationRenderer.java @@ -260,12 +260,18 @@ public List render(RenderRequest aRequest, List aFea labelFeatures); case WHEN_SELECTED: if (aRequest.getState() == null || isSelected(aRequest, aFS, sourceFs, targetFs)) { + // State == null is when we render for the annotation sidebar... return renderRelationAsArcs(aRequest, aVDocument, aFS, typeAdapter, sourceFs, targetFs, labelFeatures); } return renderRelationOnLabel(aVDocument, typeAdapter, sourceFs, targetFs, labelFeatures); case NEVER: + if (aRequest.getState() == null) { + // State == null is when we render for the annotation sidebar... + return renderRelationAsArcs(aRequest, aVDocument, aFS, typeAdapter, sourceFs, + targetFs, labelFeatures); + } return renderRelationOnLabel(aVDocument, typeAdapter, sourceFs, targetFs, labelFeatures); default: From 9d4a8ad0283de72fcb81adca7f1adbfc50f97c53 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 6 Nov 2024 21:19:22 +0100 Subject: [PATCH 051/106] #5149 - Page hangs when listing LLM models is not possible - Set a 10 second timeout for model listing calls --- .../recommendation/imls/chatgpt/client/ChatGptClientImpl.java | 2 ++ .../recommendation/imls/hf/client/HfHubClientImpl.java | 2 ++ .../recommendation/imls/ollama/client/OllamaClientImpl.java | 2 ++ 3 files changed, 6 insertions(+) diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/client/ChatGptClientImpl.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/client/ChatGptClientImpl.java index b4c47743dd6..c5bba54eefc 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/client/ChatGptClientImpl.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/client/ChatGptClientImpl.java @@ -29,6 +29,7 @@ import java.net.http.HttpRequest; import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; +import java.time.Duration; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -141,6 +142,7 @@ public List listModels(String aUrl, ListModelsRequest aRequest) th .uri(URI.create(appendIfMissing(aUrl, "/") + "models")) // .header(HttpHeaders.CONTENT_TYPE, "application/json").GET() // .header("Authorization", "Bearer " + aRequest.getApiKey()) // + .timeout(Duration.ofSeconds(10)) // .build(); var response = sendRequest(request); diff --git a/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java b/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java index 408a530f62e..bf9813ce215 100644 --- a/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java +++ b/inception/inception-imls-hf/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/hf/client/HfHubClientImpl.java @@ -24,6 +24,7 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; @@ -82,6 +83,7 @@ public HfModelCard[] queryCatalog(Map aQueryParameters) throws I HttpRequest request = HttpRequest.newBuilder() // .uri(URI.create(uriBuilder.toString())) // + .timeout(Duration.ofSeconds(10)) // .build(); HttpResponse response = sendRequest(request); diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImpl.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImpl.java index e229c667681..2cb4b27ef16 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImpl.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/client/OllamaClientImpl.java @@ -30,6 +30,7 @@ import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -142,6 +143,7 @@ public List listModels(String aUrl) throws IOException var request = HttpRequest.newBuilder() // .uri(URI.create(appendIfMissing(aUrl, "/") + "api/tags")) // .header(HttpHeaders.CONTENT_TYPE, "application/json").GET() // + .timeout(Duration.ofSeconds(10)) // .build(); var response = sendRequest(request); From 2b02c33e31f4bcd7eed704b18d7c44ed43a289da Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Thu, 7 Nov 2024 08:49:12 +0100 Subject: [PATCH 052/106] #5152 - Formatting broken when opening TEI files - Ensure resources are included in TEI IO module - Exclude some files from rat check with fail to pass sometimes (probably bug in rat) --- inception/inception-io-tei/marker-wicket-module | 1 + inception/inception-pdf-editor/pom.xml | 2 ++ inception/inception-tutorial/pom.xml | 14 ++++++++++++++ 3 files changed, 17 insertions(+) create mode 100644 inception/inception-io-tei/marker-wicket-module diff --git a/inception/inception-io-tei/marker-wicket-module b/inception/inception-io-tei/marker-wicket-module new file mode 100644 index 00000000000..44dcaf8bea4 --- /dev/null +++ b/inception/inception-io-tei/marker-wicket-module @@ -0,0 +1 @@ +Marker file which activates the profile "wicket-module" from the parent POM. diff --git a/inception/inception-pdf-editor/pom.xml b/inception/inception-pdf-editor/pom.xml index f468ea23cce..a6eb3263026 100644 --- a/inception/inception-pdf-editor/pom.xml +++ b/inception/inception-pdf-editor/pom.xml @@ -201,6 +201,8 @@ src/main/java/de/tudarmstadt/ukp/inception/pdfeditor/pdfextract/PDFExtractor.java src/main/java/de/tudarmstadt/ukp/inception/pdfeditor/pdfextract/ImageExtractor.java + src/main/java/de/tudarmstadt/ukp/inception/pdfeditor/pdfextract/DrawOperator.java + src/main/java/de/tudarmstadt/ukp/inception/pdfeditor/pdfextract/TextOperator.java src/main/ts/**/* src/main/ts/**/* diff --git a/inception/inception-tutorial/pom.xml b/inception/inception-tutorial/pom.xml index 20eef59fc03..5a9e0924726 100644 --- a/inception/inception-tutorial/pom.xml +++ b/inception/inception-tutorial/pom.xml @@ -88,6 +88,20 @@ + + org.apache.rat + apache-rat-plugin + + + default + + + src/main/java/de/tudarmstadt/ukp/inception/tutorial/enjoyhint.js + + + + + org.apache.maven.plugins maven-dependency-plugin From 77ae8acb51de5f8a263d546c15b91cd2ec30c747 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Wed, 6 Nov 2024 19:56:30 +0100 Subject: [PATCH 053/106] #5140 - Support access to tag sets in prompt template - Add tagset access to the LLM recommenders --- .../AzureAiOpenAiRecommender.java | 57 ++------- .../AzureAiOpenAiRecommenderFactory.java | 8 +- .../AzureAiOpenAiRecommenderTraits.java | 77 +----------- ...eAiOpenAiRecommenderAutoConfiguration.java | 5 +- .../imls/chatgpt/ChatGptRecommender.java | 54 +-------- .../chatgpt/ChatGptRecommenderFactory.java | 7 +- .../chatgpt/ChatGptRecommenderTraits.java | 71 +---------- .../ChatGptRecommenderAutoConfiguration.java | 6 +- .../recommendation/imls/chatgpt/presets.yaml | 43 +++++-- .../imls/ollama/OllamaRecommender.java | 56 +-------- .../imls/ollama/OllamaRecommenderFactory.java | 7 +- .../imls/ollama/OllamaRecommenderTraits.java | 77 +----------- .../OllamaRecommenderAutoConfiguration.java | 6 +- .../imls/ollama/OllamaRecommenderTest.java | 17 ++- inception/inception-imls-support-llm/pom.xml | 9 +- .../llm/option/LlmRecommenderImplBase.java | 110 ++++++++++++++++++ .../llm/option/LlmRecommenderTraits.java | 99 ++++++++++++++++ .../llm/prompt/PromptContextGenerator.java | 1 + .../support/llm/prompt/PromptingMode.java | 11 +- 19 files changed, 316 insertions(+), 405 deletions(-) create mode 100644 inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderImplBase.java create mode 100644 inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderTraits.java diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommender.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommender.java index 1f6a7027b61..34a2c89d3a6 100644 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommender.java +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommender.java @@ -17,80 +17,37 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.VAR_EXAMPLES; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.getPromptContextGenerator; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ResponseExtractor.getResponseExtractor; - import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.Map; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.uima.cas.CAS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.NonTrainableRecommenderEngineImplBase; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; import de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.client.AzureAiOpenAiClient; import de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.client.ChatCompletionRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.JinjaPromptRenderer; -import de.tudarmstadt.ukp.inception.rendering.model.Range; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option.LlmRecommenderImplBase; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; -import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public class AzureAiOpenAiRecommender - extends NonTrainableRecommenderEngineImplBase + extends LlmRecommenderImplBase { - private static final int MAX_FEW_SHOT_EXAMPLES = 10; - private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final AzureAiOpenAiRecommenderTraits traits; - private final AzureAiOpenAiClient client; - private final JinjaPromptRenderer promptRenderer; public AzureAiOpenAiRecommender(Recommender aRecommender, - AzureAiOpenAiRecommenderTraits aTraits, AzureAiOpenAiClient aClient) + AzureAiOpenAiRecommenderTraits aTraits, AzureAiOpenAiClient aClient, + AnnotationSchemaService aSchemaService) { - super(aRecommender); + super(aRecommender, aTraits, aSchemaService); - traits = aTraits; client = aClient; - promptRenderer = new JinjaPromptRenderer(); } @Override - public Range predict(PredictionContext aContext, CAS aCas, int aBegin, int aEnd) - throws RecommendationException - { - var responseExtractor = getResponseExtractor(traits.getExtractionMode()); - var examples = responseExtractor.generate(this, aCas, MAX_FEW_SHOT_EXAMPLES); - var globalBindings = Map.of(VAR_EXAMPLES, examples); - - getPromptContextGenerator(traits.getPromptingMode()) - .generate(this, aCas, aBegin, aEnd, globalBindings).forEach(promptContext -> { - try { - var prompt = promptRenderer.render(traits.getPrompt(), promptContext); - var response = query(prompt); - responseExtractor.extract(this, aCas, promptContext, response); - } - catch (IOException e) { - aContext.log(LogMessage.warn(getRecommender().getName(), - "Azure AI OpenAI failed to respond: %s", - ExceptionUtils.getRootCauseMessage(e))); - LOG.error("Azure AI OpenAI failed to respond: {}", - ExceptionUtils.getRootCauseMessage(e)); - } - }); - - return new Range(aBegin, aEnd); - } - - private String query(String aPrompt) throws IOException + protected String query(String aPrompt) throws IOException { LOG.trace("Querying Azure AI OpenAI: [{}]", aPrompt); var request = ChatCompletionRequest.builder() // diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderFactory.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderFactory.java index 669ac40f2e6..9fd7b981a4e 100644 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderFactory.java +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderFactory.java @@ -39,6 +39,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactoryImplBase; import de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.client.AzureAiOpenAiClient; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.io.WatchedResourceFile; import de.tudarmstadt.ukp.inception.support.yaml.YamlUtil; import de.tudarmstadt.ukp.inception.ui.core.docanno.layer.DocumentMetadataLayerSupport; @@ -53,12 +54,15 @@ public class AzureAiOpenAiRecommenderFactory private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final AzureAiOpenAiClient client; + private final AnnotationSchemaService schemaService; private WatchedResourceFile> presets; - public AzureAiOpenAiRecommenderFactory(AzureAiOpenAiClient aClient) + public AzureAiOpenAiRecommenderFactory(AzureAiOpenAiClient aClient, + AnnotationSchemaService aSchemaService) { client = aClient; + schemaService = aSchemaService; var presetsResource = getClass().getResource("presets.yaml"); presets = new WatchedResourceFile<>(presetsResource, is -> YamlUtil.getObjectMapper() @@ -83,7 +87,7 @@ public String getName() public RecommendationEngine build(Recommender aRecommender) { AzureAiOpenAiRecommenderTraits traits = readTraits(aRecommender); - return new AzureAiOpenAiRecommender(aRecommender, traits, client); + return new AzureAiOpenAiRecommender(aRecommender, traits, client, schemaService); } @Override diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraits.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraits.java index 9ce1818813b..da7b4a28c79 100644 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraits.java +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraits.java @@ -17,24 +17,12 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai; -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY; - -import java.io.Serializable; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; - import de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.client.GenerateResponseFormat; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptingMode; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ExtractionMode; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option.LlmRecommenderTraits; import de.tudarmstadt.ukp.inception.security.client.auth.AuthenticationTraits; -@JsonIgnoreProperties(ignoreUnknown = true) public class AzureAiOpenAiRecommenderTraits - implements Serializable + extends LlmRecommenderTraits { private static final long serialVersionUID = 6433061638746045602L; @@ -44,18 +32,8 @@ public class AzureAiOpenAiRecommenderTraits private AuthenticationTraits authentication; - private String prompt; - private GenerateResponseFormat format; - private PromptingMode promptingMode = PromptingMode.PER_ANNOTATION; - - private ExtractionMode extractionMode = ExtractionMode.RESPONSE_AS_LABEL; - - private @JsonInclude(NON_EMPTY) Map options = new LinkedHashMap(); - - private boolean interactive; - public AuthenticationTraits getAuthentication() { return authentication; @@ -76,26 +54,6 @@ public void setUrl(String aUrl) url = aUrl; } - public String getPrompt() - { - return prompt; - } - - public void setPrompt(String aPrompt) - { - prompt = aPrompt; - } - - public PromptingMode getPromptingMode() - { - return promptingMode; - } - - public void setPromptingMode(PromptingMode aPromptingMode) - { - promptingMode = aPromptingMode; - } - public GenerateResponseFormat getFormat() { return format; @@ -105,35 +63,4 @@ public void setFormat(GenerateResponseFormat aFormat) { format = aFormat; } - - public ExtractionMode getExtractionMode() - { - return extractionMode; - } - - public void setExtractionMode(ExtractionMode aExtractionMode) - { - extractionMode = aExtractionMode; - } - - public Map getOptions() - { - return Collections.unmodifiableMap(options); - } - - public void setOptions(Map aOptions) - { - options.clear(); - options.putAll(aOptions); - } - - public boolean isInteractive() - { - return interactive; - } - - public void setInteractive(boolean aInteractive) - { - interactive = aInteractive; - } } diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java index 59ac6924257..733d97d9a13 100644 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java +++ b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java @@ -24,6 +24,7 @@ import de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.AzureAiOpenAiRecommenderFactory; import de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.client.AzureAiOpenAiClient; import de.tudarmstadt.ukp.inception.recommendation.imls.azureaiopenai.client.AzureAiOpenAiClientImpl; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @Configuration @ConditionalOnProperty(prefix = "recommender.azureai-openai", name = "enabled", // @@ -38,8 +39,8 @@ public AzureAiOpenAiClient azureAiOpenAiClient() @Bean public AzureAiOpenAiRecommenderFactory azureAiOpenAiRecommenderFactory( - AzureAiOpenAiClient aClient) + AzureAiOpenAiClient aClient, AnnotationSchemaService aSchemaService) { - return new AzureAiOpenAiRecommenderFactory(aClient); + return new AzureAiOpenAiRecommenderFactory(aClient, aSchemaService); } } diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommender.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommender.java index f7132b260a6..b98d177b9d0 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommender.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommender.java @@ -17,79 +17,37 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.VAR_EXAMPLES; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.getPromptContextGenerator; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ResponseExtractor.getResponseExtractor; -import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; - import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.Map; -import org.apache.uima.cas.CAS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.NonTrainableRecommenderEngineImplBase; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.client.ChatCompletionRequest; import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.client.ChatGptClient; import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.client.ResponseFormat; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.JinjaPromptRenderer; -import de.tudarmstadt.ukp.inception.rendering.model.Range; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option.LlmRecommenderImplBase; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.security.client.auth.apikey.ApiKeyAuthenticationTraits; -import de.tudarmstadt.ukp.inception.support.logging.LogMessage; public class ChatGptRecommender - extends NonTrainableRecommenderEngineImplBase + extends LlmRecommenderImplBase { - private static final int MAX_FEW_SHOT_EXAMPLES = 10; - private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final ChatGptRecommenderTraits traits; - private final ChatGptClient client; - private final JinjaPromptRenderer promptRenderer; public ChatGptRecommender(Recommender aRecommender, ChatGptRecommenderTraits aTraits, - ChatGptClient aClient) + ChatGptClient aClient, AnnotationSchemaService aSchemaService) { - super(aRecommender); + super(aRecommender, aTraits, aSchemaService); - traits = aTraits; client = aClient; - promptRenderer = new JinjaPromptRenderer(); } @Override - public Range predict(PredictionContext aContext, CAS aCas, int aBegin, int aEnd) - throws RecommendationException - { - var responseExtractor = getResponseExtractor(traits.getExtractionMode()); - var examples = responseExtractor.generate(this, aCas, MAX_FEW_SHOT_EXAMPLES); - var globalBindings = Map.of(VAR_EXAMPLES, examples); - - getPromptContextGenerator(traits.getPromptingMode()) - .generate(this, aCas, aBegin, aEnd, globalBindings).forEach(promptContext -> { - try { - var prompt = promptRenderer.render(traits.getPrompt(), promptContext); - var response = query(prompt); - responseExtractor.extract(this, aCas, promptContext, response); - } - catch (IOException e) { - aContext.log(LogMessage.warn(getRecommender().getName(), - "Remote failed to respond: %s", getRootCauseMessage(e))); - LOG.error("Remote failed to respond: {}", getRootCauseMessage(e)); - } - }); - - return new Range(aBegin, aEnd); - } - - private String query(String aPrompt) throws IOException + protected String query(String aPrompt) throws IOException { LOG.trace("Query: [{}]", aPrompt); var request = ChatCompletionRequest.builder() // diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderFactory.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderFactory.java index 1442e35c6d7..9ae0f15866b 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderFactory.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderFactory.java @@ -39,6 +39,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactoryImplBase; import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.client.ChatGptClient; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.io.WatchedResourceFile; import de.tudarmstadt.ukp.inception.support.yaml.YamlUtil; import de.tudarmstadt.ukp.inception.ui.core.docanno.layer.DocumentMetadataLayerSupport; @@ -53,12 +54,14 @@ public class ChatGptRecommenderFactory private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final ChatGptClient client; + private final AnnotationSchemaService schemaService; private WatchedResourceFile> presets; - public ChatGptRecommenderFactory(ChatGptClient aClient) + public ChatGptRecommenderFactory(ChatGptClient aClient, AnnotationSchemaService aSchemaService) { client = aClient; + schemaService = aSchemaService; var presetsResource = getClass().getResource("presets.yaml"); presets = new WatchedResourceFile<>(presetsResource, is -> YamlUtil.getObjectMapper() @@ -83,7 +86,7 @@ public String getName() public RecommendationEngine build(Recommender aRecommender) { ChatGptRecommenderTraits traits = readTraits(aRecommender); - return new ChatGptRecommender(aRecommender, traits, client); + return new ChatGptRecommender(aRecommender, traits, client, schemaService); } @Override diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderTraits.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderTraits.java index 27f6c66b82f..3e12c4069a6 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderTraits.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderTraits.java @@ -17,23 +17,17 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt; -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY; - import java.io.Serializable; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.client.ResponseFormatType; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptingMode; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ExtractionMode; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option.LlmRecommenderTraits; import de.tudarmstadt.ukp.inception.security.client.auth.AuthenticationTraits; @JsonIgnoreProperties(ignoreUnknown = true) public class ChatGptRecommenderTraits + extends LlmRecommenderTraits implements Serializable { private static final long serialVersionUID = 6433061638746045602L; @@ -47,20 +41,10 @@ public class ChatGptRecommenderTraits private AuthenticationTraits authentication; - private String prompt; - private String model = "gpt-3.5-turbo"; private ResponseFormatType format; - private PromptingMode promptingMode = PromptingMode.PER_ANNOTATION; - - private ExtractionMode extractionMode = ExtractionMode.RESPONSE_AS_LABEL; - - private @JsonInclude(NON_EMPTY) Map options = new LinkedHashMap(); - - private boolean interactive; - public AuthenticationTraits getAuthentication() { return authentication; @@ -91,26 +75,6 @@ public void setModel(String aModel) model = aModel; } - public String getPrompt() - { - return prompt; - } - - public void setPrompt(String aPrompt) - { - prompt = aPrompt; - } - - public PromptingMode getPromptingMode() - { - return promptingMode; - } - - public void setPromptingMode(PromptingMode aPromptingMode) - { - promptingMode = aPromptingMode; - } - public ResponseFormatType getFormat() { return format; @@ -120,35 +84,4 @@ public void setFormat(ResponseFormatType aFormat) { format = aFormat; } - - public ExtractionMode getExtractionMode() - { - return extractionMode; - } - - public void setExtractionMode(ExtractionMode aExtractionMode) - { - extractionMode = aExtractionMode; - } - - public Map getOptions() - { - return Collections.unmodifiableMap(options); - } - - public void setOptions(Map aOptions) - { - options.clear(); - options.putAll(aOptions); - } - - public boolean isInteractive() - { - return interactive; - } - - public void setInteractive(boolean aInteractive) - { - interactive = aInteractive; - } } diff --git a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/config/ChatGptRecommenderAutoConfiguration.java b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/config/ChatGptRecommenderAutoConfiguration.java index d8d9228091d..d4e088cfc2a 100644 --- a/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/config/ChatGptRecommenderAutoConfiguration.java +++ b/inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/config/ChatGptRecommenderAutoConfiguration.java @@ -24,6 +24,7 @@ import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.ChatGptRecommenderFactory; import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.client.ChatGptClient; import de.tudarmstadt.ukp.inception.recommendation.imls.chatgpt.client.ChatGptClientImpl; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @Configuration @ConditionalOnProperty(prefix = "recommender.chatgpt", name = "enabled", // @@ -37,8 +38,9 @@ public ChatGptClient chatGptClient() } @Bean - public ChatGptRecommenderFactory chatGptRecommenderFactory(ChatGptClient aClient) + public ChatGptRecommenderFactory chatGptRecommenderFactory(ChatGptClient aClient, + AnnotationSchemaService aSchemaService) { - return new ChatGptRecommenderFactory(aClient); + return new ChatGptRecommenderFactory(aClient, aSchemaService); } } diff --git a/inception/inception-imls-chatgpt/src/main/resources/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/presets.yaml b/inception/inception-imls-chatgpt/src/main/resources/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/presets.yaml index 8fb1a4f1119..dad3d2a23be 100644 --- a/inception/inception-imls-chatgpt/src/main/resources/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/presets.yaml +++ b/inception/inception-imls-chatgpt/src/main/resources/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/presets.yaml @@ -5,33 +5,48 @@ prompt: |- Identify all named entities in the following text. + {% if tags %} + Label each entity using one of the following labels: + {% for tag, description in tags.items() %} + * {{- tag -}} {%if description %}: {{- description -}}{% endif %}{% endfor %} + {% endif %} + + Text: + ``` {{ text }} + ``` - name: Extract named entities from sentenes (dynamic few-shot) promptingMode: per-sentence format: json extractionMode: mentions-from-json prompt: |- - Identify all named entities in the following text. - + Identify all named entities in the following text and return them as JSON. + + {% if tags %} + Label each entity using one of the following labels: + {% for tag, description in tags.items() %} + * {{- tag -}} {%if description %}: {{- description -}}{% endif %}{% endfor %} + {% endif %} + {% if examples %} {% for example in examples %} Text: - ''' + ``` {{ example.getText() }} - ''' - + ``` + Response: {{ example.getLabelledMentions() | tojson }} {% endfor %} - + Text: {% endif %} - - ''' + + ``` {{ text }} - ''' - + ``` + {% if examples %} Response: {% endif %} @@ -41,8 +56,9 @@ promptingMode: per-sentence extractionMode: response-as-label prompt: |- - Summarize the following sentence in a single word. + Summarize the following text in a single word. + Text: ``` {{ text }} ``` @@ -53,6 +69,7 @@ prompt: |- Briefly describe what the following text is about. + Text: ``` {{ text }} ``` @@ -63,6 +80,7 @@ prompt: |- Briefly summarize the following text. + Text: ``` {% for x in cas.select('custom.Span') %} {{ x }} @@ -73,8 +91,9 @@ promptingMode: per-annotation extractionMode: response-as-label prompt: |- - Very briefly describe the meaning of `{{ text }}` in the following sentence. + Very briefly describe the meaning of `{{ text }}` in the following text. + Text: ``` {{ sentence }} ``` diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java index 8770d8f6d27..7e2d57dafdc 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommender.java @@ -17,81 +17,37 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.ollama; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.VAR_EXAMPLES; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.getPromptContextGenerator; -import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ResponseExtractor.getResponseExtractor; import static java.lang.System.currentTimeMillis; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.util.Map; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.apache.uima.cas.CAS; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.NonTrainableRecommenderEngineImplBase; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; -import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client.OllamaClient; import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client.OllamaGenerateRequest; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.JinjaPromptRenderer; -import de.tudarmstadt.ukp.inception.rendering.model.Range; -import de.tudarmstadt.ukp.inception.support.logging.LogMessage; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option.LlmRecommenderImplBase; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; public class OllamaRecommender - extends NonTrainableRecommenderEngineImplBase + extends LlmRecommenderImplBase { - private static final int MAX_FEW_SHOT_EXAMPLES = 10; - private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final OllamaRecommenderTraits traits; - private final OllamaClient client; - private final JinjaPromptRenderer promptRenderer; - public OllamaRecommender(Recommender aRecommender, OllamaRecommenderTraits aTraits, - OllamaClient aClient) + OllamaClient aClient, AnnotationSchemaService aSchemaService) { - super(aRecommender); + super(aRecommender, aTraits, aSchemaService); - traits = aTraits; client = aClient; - promptRenderer = new JinjaPromptRenderer(); } @Override - public Range predict(PredictionContext aContext, CAS aCas, int aBegin, int aEnd) - throws RecommendationException - { - var responseExtractor = getResponseExtractor(traits.getExtractionMode()); - var examples = responseExtractor.generate(this, aCas, MAX_FEW_SHOT_EXAMPLES); - var globalBindings = Map.of(VAR_EXAMPLES, examples); - - getPromptContextGenerator(traits.getPromptingMode()) - .generate(this, aCas, aBegin, aEnd, globalBindings).forEach(promptContext -> { - try { - var prompt = promptRenderer.render(traits.getPrompt(), promptContext); - var response = query(prompt); - responseExtractor.extract(this, aCas, promptContext, response); - } - catch (IOException e) { - aContext.log(LogMessage.warn(getRecommender().getName(), - "Ollama [%s] failed to respond: %s", traits.getModel(), - ExceptionUtils.getRootCauseMessage(e))); - LOG.error("Ollama [{}] failed to respond: {}", traits.getModel(), - ExceptionUtils.getRootCauseMessage(e)); - } - }); - - return new Range(aBegin, aEnd); - } - - private String query(String aPrompt) throws IOException + protected String query(String aPrompt) throws IOException { LOG.trace("Querying ollama [{}]: [{}]", traits.getModel(), aPrompt); var request = OllamaGenerateRequest.builder() // diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderFactory.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderFactory.java index 251b15d5b5e..1ec9b718b6e 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderFactory.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderFactory.java @@ -39,6 +39,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngine; import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationEngineFactoryImplBase; import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client.OllamaClient; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.io.WatchedResourceFile; import de.tudarmstadt.ukp.inception.support.yaml.YamlUtil; import de.tudarmstadt.ukp.inception.ui.core.docanno.layer.DocumentMetadataLayerSupport; @@ -52,13 +53,15 @@ public class OllamaRecommenderFactory private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final AnnotationSchemaService schemaService; private final OllamaClient client; private WatchedResourceFile> presets; - public OllamaRecommenderFactory(OllamaClient aClient) + public OllamaRecommenderFactory(OllamaClient aClient, AnnotationSchemaService aSchemaService) { client = aClient; + schemaService = aSchemaService; var presetsResource = getClass().getResource("presets.yaml"); presets = new WatchedResourceFile<>(presetsResource, is -> YamlUtil.getObjectMapper() @@ -83,7 +86,7 @@ public String getName() public RecommendationEngine build(Recommender aRecommender) { OllamaRecommenderTraits traits = readTraits(aRecommender); - return new OllamaRecommender(aRecommender, traits, client); + return new OllamaRecommender(aRecommender, traits, client, schemaService); } @Override diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTraits.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTraits.java index cf451a32a3c..c5ae2c516e3 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTraits.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTraits.java @@ -17,23 +17,11 @@ */ package de.tudarmstadt.ukp.inception.recommendation.imls.ollama; -import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY; - -import java.io.Serializable; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; - import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client.OllamaGenerateResponseFormat; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptingMode; -import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ExtractionMode; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option.LlmRecommenderTraits; -@JsonIgnoreProperties(ignoreUnknown = true) public class OllamaRecommenderTraits - implements Serializable + extends LlmRecommenderTraits { public static final String DEFAULT_OLLAMA_URL = "http://localhost:11434/"; @@ -43,20 +31,10 @@ public class OllamaRecommenderTraits private String model; - private String prompt; - private boolean raw; private OllamaGenerateResponseFormat format; - private PromptingMode promptingMode = PromptingMode.PER_ANNOTATION; - - private ExtractionMode extractionMode = ExtractionMode.RESPONSE_AS_LABEL; - - private @JsonInclude(NON_EMPTY) Map options = new LinkedHashMap(); - - private boolean interactive; - public String getUrl() { return url; @@ -77,26 +55,6 @@ public void setModel(String aModel) model = aModel; } - public String getPrompt() - { - return prompt; - } - - public void setPrompt(String aPrompt) - { - prompt = aPrompt; - } - - public PromptingMode getPromptingMode() - { - return promptingMode; - } - - public void setPromptingMode(PromptingMode aPromptingMode) - { - promptingMode = aPromptingMode; - } - public boolean isRaw() { return raw; @@ -116,35 +74,4 @@ public void setFormat(OllamaGenerateResponseFormat aFormat) { format = aFormat; } - - public ExtractionMode getExtractionMode() - { - return extractionMode; - } - - public void setExtractionMode(ExtractionMode aExtractionMode) - { - extractionMode = aExtractionMode; - } - - public Map getOptions() - { - return Collections.unmodifiableMap(options); - } - - public void setOptions(Map aOptions) - { - options.clear(); - options.putAll(aOptions); - } - - public boolean isInteractive() - { - return interactive; - } - - public void setInteractive(boolean aInteractive) - { - interactive = aInteractive; - } } diff --git a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/config/OllamaRecommenderAutoConfiguration.java b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/config/OllamaRecommenderAutoConfiguration.java index 0494d01415f..8abe752c8f3 100644 --- a/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/config/OllamaRecommenderAutoConfiguration.java +++ b/inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/config/OllamaRecommenderAutoConfiguration.java @@ -24,6 +24,7 @@ import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.OllamaRecommenderFactory; import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client.OllamaClient; import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client.OllamaClientImpl; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; @Configuration @ConditionalOnProperty(prefix = "recommender.ollama", name = "enabled", havingValue = "true", matchIfMissing = false) @@ -36,8 +37,9 @@ public OllamaClient ollamaClient() } @Bean - public OllamaRecommenderFactory ollamaRecommenderFactory(OllamaClient aClient) + public OllamaRecommenderFactory ollamaRecommenderFactory(OllamaClient aClient, + AnnotationSchemaService aSchemaService) { - return new OllamaRecommenderFactory(aClient); + return new OllamaRecommenderFactory(aClient, aSchemaService); } } diff --git a/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java index 6a5fe726fbf..42ae9882d5a 100644 --- a/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java +++ b/inception/inception-imls-ollama/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTest.java @@ -38,6 +38,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,12 +55,16 @@ import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommenderContext; import de.tudarmstadt.ukp.inception.recommendation.imls.ollama.client.OllamaClientImpl; import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ExtractionMode; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; import de.tudarmstadt.ukp.inception.support.test.http.HttpTestUtils; +@ExtendWith(MockitoExtension.class) class OllamaRecommenderTest { private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private @Mock AnnotationSchemaService schemaSerivce; + private AnnotationLayer layer; private AnnotationFeature feature; private Recommender recommender; @@ -96,7 +103,7 @@ void testPerDocumentUsingReponseAsLabel() throws Exception traits.setPromptingMode(PER_DOCUMENT); traits.setExtractionMode(ExtractionMode.RESPONSE_AS_LABEL); - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl()); + var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce); sut.predict(new PredictionContext(new RecommenderContext()), cas); var predictions = cas.select(NamedEntity.class) @@ -121,7 +128,7 @@ void testPerDocumentUsingMentionsFromJsonList_Numbers() throws Exception traits.setPromptingMode(PER_DOCUMENT); traits.setExtractionMode(MENTIONS_FROM_JSON); - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl()); + var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce); sut.predict(new PredictionContext(new RecommenderContext()), cas); var predictions = cas.select(NamedEntity.class) @@ -144,7 +151,7 @@ void testPerDocumentUsingMentionsFromJsonList_Entities() throws Exception traits.setPromptingMode(PER_DOCUMENT); traits.setExtractionMode(MENTIONS_FROM_JSON); - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl()); + var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce); sut.predict(new PredictionContext(new RecommenderContext()), cas); var predictions = cas.select(NamedEntity.class) @@ -172,7 +179,7 @@ void testPerDocumentUsingMentionsFromJsonList_Politicians() throws Exception traits.setPromptingMode(PER_DOCUMENT); traits.setExtractionMode(MENTIONS_FROM_JSON); - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl()); + var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce); sut.predict(new PredictionContext(new RecommenderContext()), cas); var predictions = cas.select(NamedEntity.class) @@ -222,7 +229,7 @@ void testPerSentenceUsingMentionsFromJsonList_Politicians_fewShjot() throws Exce traits.setPromptingMode(PER_SENTENCE); traits.setExtractionMode(MENTIONS_FROM_JSON); - var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl()); + var sut = new OllamaRecommender(recommender, traits, new OllamaClientImpl(), schemaSerivce); sut.predict(new PredictionContext(new RecommenderContext()), cas); var predictions = cas.select(NamedEntity.class) diff --git a/inception/inception-imls-support-llm/pom.xml b/inception/inception-imls-support-llm/pom.xml index b3b97ff11a6..3a4d7c6be0c 100644 --- a/inception/inception-imls-support-llm/pom.xml +++ b/inception/inception-imls-support-llm/pom.xml @@ -48,6 +48,10 @@ de.tudarmstadt.ukp.inception.app inception-support + + de.tudarmstadt.ukp.inception.app + inception-api-render + org.apache.commons @@ -88,11 +92,6 @@ slf4j-api - - de.tudarmstadt.ukp.inception.app - inception-api-render - test - org.dkpro.core dkpro-core-api-ner-asl diff --git a/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderImplBase.java b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderImplBase.java new file mode 100644 index 00000000000..5984088ff3f --- /dev/null +++ b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderImplBase.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option; + +import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.VAR_EXAMPLES; +import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.VAR_TAGS; +import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptContextGenerator.getPromptContextGenerator; +import static de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ResponseExtractor.getResponseExtractor; +import static java.util.stream.Collectors.toMap; +import static org.apache.commons.lang3.exception.ExceptionUtils.getRootCauseMessage; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +import org.apache.uima.cas.CAS; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import de.tudarmstadt.ukp.inception.recommendation.api.model.Recommender; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.NonTrainableRecommenderEngineImplBase; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.PredictionContext; +import de.tudarmstadt.ukp.inception.recommendation.api.recommender.RecommendationException; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.JinjaPromptRenderer; +import de.tudarmstadt.ukp.inception.rendering.model.Range; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; +import de.tudarmstadt.ukp.inception.support.logging.LogMessage; + +public abstract class LlmRecommenderImplBase + extends NonTrainableRecommenderEngineImplBase +{ + private static final int MAX_FEW_SHOT_EXAMPLES = 10; + + private final static Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected final T traits; + protected final AnnotationSchemaService schemaService; + protected final JinjaPromptRenderer promptRenderer; + + public LlmRecommenderImplBase(Recommender aRecommender, T aTraits, + AnnotationSchemaService aSchemaService) + { + super(aRecommender); + + traits = aTraits; + schemaService = aSchemaService; + promptRenderer = new JinjaPromptRenderer(); + } + + protected Map prepareGlobalBindings(CAS aCas) + { + var globalBindings = new LinkedHashMap(); + var responseExtractor = getResponseExtractor(traits.getExtractionMode()); + var examples = responseExtractor.generate(this, aCas, MAX_FEW_SHOT_EXAMPLES); + globalBindings.put(VAR_EXAMPLES, examples); + + var tagset = getRecommender().getFeature().getTagset(); + if (tagset != null) { + var tags = schemaService.listTags(tagset).stream() // + .collect(toMap( // + tag -> tag.getName(), // + tag -> Objects.toString(tag.getDescription(), ""))); + globalBindings.put(VAR_TAGS, tags); + } + return globalBindings; + } + + @Override + public Range predict(PredictionContext aContext, CAS aCas, int aBegin, int aEnd) + throws RecommendationException + { + var globalBindings = prepareGlobalBindings(aCas); + + var responseExtractor = getResponseExtractor(traits.getExtractionMode()); + getPromptContextGenerator(traits.getPromptingMode()) + .generate(this, aCas, aBegin, aEnd, globalBindings).forEach(promptContext -> { + try { + var prompt = promptRenderer.render(traits.getPrompt(), promptContext); + var response = query(prompt); + responseExtractor.extract(this, aCas, promptContext, response); + } + catch (IOException e) { + aContext.log(LogMessage.warn(getRecommender().getName(), + "Remote failed to respond: %s", getRootCauseMessage(e))); + LOG.error("Remote failed to respond: {}", getRootCauseMessage(e)); + } + }); + + return new Range(aBegin, aEnd); + } + + protected abstract String query(String aPrompt) throws IOException; +} diff --git a/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderTraits.java b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderTraits.java new file mode 100644 index 00000000000..9ca51f894be --- /dev/null +++ b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option/LlmRecommenderTraits.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.option; + +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_EMPTY; + +import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.prompt.PromptingMode; +import de.tudarmstadt.ukp.inception.recommendation.imls.support.llm.response.ExtractionMode; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class LlmRecommenderTraits + implements Serializable +{ + private static final long serialVersionUID = 6433061638746045602L; + + private String prompt; + + private PromptingMode promptingMode = PromptingMode.PER_ANNOTATION; + + private ExtractionMode extractionMode = ExtractionMode.RESPONSE_AS_LABEL; + + private @JsonInclude(NON_EMPTY) Map options = new LinkedHashMap(); + + private boolean interactive; + + public String getPrompt() + { + return prompt; + } + + public void setPrompt(String aPrompt) + { + prompt = aPrompt; + } + + public PromptingMode getPromptingMode() + { + return promptingMode; + } + + public void setPromptingMode(PromptingMode aPromptingMode) + { + promptingMode = aPromptingMode; + } + + public ExtractionMode getExtractionMode() + { + return extractionMode; + } + + public void setExtractionMode(ExtractionMode aExtractionMode) + { + extractionMode = aExtractionMode; + } + + public Map getOptions() + { + return Collections.unmodifiableMap(options); + } + + public void setOptions(Map aOptions) + { + options.clear(); + options.putAll(aOptions); + } + + public boolean isInteractive() + { + return interactive; + } + + public void setInteractive(boolean aInteractive) + { + interactive = aInteractive; + } +} diff --git a/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptContextGenerator.java b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptContextGenerator.java index 3ad9c760fa3..0f88dd46cc6 100644 --- a/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptContextGenerator.java +++ b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptContextGenerator.java @@ -31,6 +31,7 @@ public interface PromptContextGenerator static final String VAR_DOCUMENT = "document"; static final String VAR_EXAMPLES = "examples"; static final String VAR_CAS = "cas"; + static final String VAR_TAGS = "tags"; Stream generate(RecommendationEngine aEngine, CAS aCas, int aBegin, int aEnd, Map aBindings); diff --git a/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptingMode.java b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptingMode.java index 89ad4e5e845..4b70729851c 100644 --- a/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptingMode.java +++ b/inception/inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/PromptingMode.java @@ -25,22 +25,25 @@ public enum PromptingMode PER_ANNOTATION(""" Template variables: - * `text`: annotation text, - * `sentence`: sentence containing annotation, + * `text`: annotation text + * `tags`: tagset (if available) + * `sentence`: sentence containing annotation * `examples`: labeled annotations"""), @JsonProperty("per-sentence") PER_SENTENCE(""" Template variables: - * `text`: sentence text, + * `text`: sentence text + * `tags`: tagset (if available) * `examples`: labeled annotations"""), @JsonProperty("per-document") PER_DOCUMENT(""" Template variables: - * `text`: document text"""); + * `text`: document text, + * `tags`: tagset (if available)"""); private final String hints; From d977e765b1b6daa2d7f8d9a831f8bebbaf1e6f9a Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 10 Nov 2024 12:02:00 +0100 Subject: [PATCH 054/106] #5140 - Support access to tag sets in prompt template - Consolidate more common code into the llm support module --- inception/inception-bom/pom.xml | 2 +- .../inception-imls-azureai-openai/pom.xml | 2 +- .../AzureAiOpenAiRecommenderTraitsEditor.html | 100 -------- .../AzureAiOpenAiRecommenderTraitsEditor.java | 151 ------------ .../imls/azureaiopenai/Preset.java | 117 ---------- .../AzureAiOpenAiInteractionPanel.html | 0 .../AzureAiOpenAiInteractionPanel.java | 11 +- .../AzureAiOpenAiRecommender.java | 21 +- .../AzureAiOpenAiRecommenderFactory.java | 5 +- .../AzureAiOpenAiRecommenderTraits.java | 29 +-- .../AzureAiOpenAiRecommenderTraitsEditor.java | 63 +++++ .../AzureAiOpenAiResponseFormatSelect.java | 4 +- .../client/AzureAiOpenAiClient.java | 2 +- .../client/AzureAiOpenAiClientImpl.java | 2 +- .../client/ChatCompletionChoice.java | 2 +- .../client/ChatCompletionMessage.java | 2 +- .../client/ChatCompletionRequest.java | 4 +- .../client/ChatCompletionResponse.java | 2 +- .../client/GenerateResponseFormat.java | 2 +- ...eAiOpenAiRecommenderAutoConfiguration.java | 8 +- ...ot.autoconfigure.AutoConfiguration.imports | 2 +- .../client/OpenAiClientTest.java | 5 +- inception/inception-imls-chatgpt/pom.xml | 2 +- .../ChatGptRecommenderTraitsEditor.html | 108 --------- .../ChatGptRecommenderTraitsEditor.java | 216 ------------------ .../chatgpt/ChatGptInteractionPanel.html | 0 .../chatgpt/ChatGptInteractionPanel.java | 13 +- .../{ => llm}/chatgpt/ChatGptRecommender.java | 20 +- .../chatgpt/ChatGptRecommenderFactory.java | 9 +- .../chatgpt/ChatGptRecommenderTraits.java | 56 +---- .../ChatGptRecommenderTraitsEditor.java | 109 +++++++++ .../chatgpt/ChatGptResponseFormatSelect.java | 4 +- .../chatgpt/client/ChatCompletionChoice.java | 2 +- .../chatgpt/client/ChatCompletionMessage.java | 2 +- .../chatgpt/client/ChatCompletionRequest.java | 4 +- .../client/ChatCompletionResponse.java | 2 +- .../chatgpt/client/ChatGptClient.java | 2 +- .../chatgpt/client/ChatGptClientImpl.java | 2 +- .../chatgpt/client/ChatGptModel.java | 2 +- .../chatgpt/client/ChatGptModelResponse.java | 2 +- .../chatgpt/client/ListModelsRequest.java | 2 +- .../chatgpt/client/ResponseFormat.java | 2 +- .../chatgpt/client/ResponseFormatType.java | 2 +- .../ChatGptRecommenderAutoConfiguration.java | 8 +- .../chatgpt/wicket-package.properties | 0 ...ot.autoconfigure.AutoConfiguration.imports | 2 +- .../client/OpenAiClientTest.java | 12 +- .../marker-wicket-module | 1 + .../pom.xml | 6 +- .../support}/prompt/AnnotationWrapper.java | 2 +- .../imls/llm/support}/prompt/CasWrapper.java | 2 +- .../support}/prompt/JinjaPromptRenderer.java | 2 +- .../prompt/PerAnnotationContextGenerator.java | 2 +- .../prompt/PerDocumentContextGenerator.java | 2 +- .../prompt/PerSentenceContextGenerator.java | 2 +- .../imls/llm/support/prompt}/Preset.java | 25 +- .../llm/support}/prompt/PromptContext.java | 2 +- .../prompt/PromptContextGenerator.java | 2 +- .../llm/support}/prompt/PromptingMode.java | 2 +- .../support}/prompt/PromptingModeSelect.java | 2 +- .../llm/support}/response/ExtractionMode.java | 2 +- .../response/ExtractionModeSelect.java | 2 +- .../response/MentionsFromJsonExtractor.java | 4 +- .../llm/support}/response/MentionsSample.java | 2 +- .../response/ResponseAsLabelExtractor.java | 4 +- .../support}/response/ResponseExtractor.java | 4 +- .../llm/support/response/ResponseFormat.java | 28 +++ .../response/ResponseFormatSelect.java} | 14 +- .../traits}/LlmRecommenderImplBase.java | 16 +- .../support/traits}/LlmRecommenderTraits.java | 56 ++++- .../LlmRecommenderTraitsEditor_ImplBase.html} | 20 +- .../LlmRecommenderTraitsEditor_ImplBase.java} | 101 ++++---- .../imls/llm/support/traits}/Option.java | 2 +- .../llm/support/traits}/OptionSetting.java | 2 +- .../imls/llm}/wicket-package.properties | 2 +- .../MentionsFromJsonExtractorTest.java | 4 +- .../support}/response/MentionsSampleTest.java | 4 +- .../support/llm/prompt/CasWrapperTest.java | 1 + .../src/test/resources/log4j2-test.xml | 0 inception/inception-imls-ollama/pom.xml | 2 +- .../ollama/OllamaInteractionPanel.html | 0 .../ollama/OllamaInteractionPanel.java | 16 +- .../{ => llm}/ollama/OllamaRecommender.java | 22 +- .../ollama/OllamaRecommenderFactory.java | 5 +- .../ollama/OllamaRecommenderTraits.java | 52 +---- .../ollama/OllamaRecommenderTraitsEditor.java | 83 +++++++ .../{ => llm}/ollama/client/OllamaClient.java | 2 +- .../ollama/client/OllamaClientImpl.java | 6 +- .../ollama/client/OllamaGenerateRequest.java | 4 +- .../ollama/client/OllamaGenerateResponse.java | 2 +- .../client/OllamaGenerateResponseFormat.java | 2 +- .../{ => llm}/ollama/client/OllamaModel.java | 2 +- .../ollama/client/OllamaTagsResponse.java | 2 +- .../OllamaRecommenderAutoConfiguration.java | 8 +- .../recommendation/imls/ollama/Preset.java | 117 ---------- .../imls/ollama/wicket-package.properties | 39 ---- ...ot.autoconfigure.AutoConfiguration.imports | 2 +- .../imls/ollama/OllamaRecommenderTest.java | 16 +- .../ollama/client/OllamaClientImplTest.java | 6 +- .../BasicRelationRecommenderInitializer.java | 6 +- .../api/RecommendationService.java | 2 +- inception/pom.xml | 2 +- 102 files changed, 625 insertions(+), 1212 deletions(-) delete mode 100644 inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraitsEditor.html delete mode 100644 inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraitsEditor.java delete mode 100644 inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/Preset.java rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/AzureAiOpenAiInteractionPanel.html (100%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/AzureAiOpenAiInteractionPanel.java (91%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/AzureAiOpenAiRecommender.java (72%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/AzureAiOpenAiRecommenderFactory.java (95%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/AzureAiOpenAiRecommenderTraits.java (67%) create mode 100644 inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/azureaiopenai/AzureAiOpenAiRecommenderTraitsEditor.java rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/AzureAiOpenAiResponseFormatSelect.java (90%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/client/AzureAiOpenAiClient.java (91%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/client/AzureAiOpenAiClientImpl.java (98%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/client/ChatCompletionChoice.java (95%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/client/ChatCompletionMessage.java (95%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/client/ChatCompletionRequest.java (95%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/client/ChatCompletionResponse.java (95%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/client/GenerateResponseFormat.java (91%) rename inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/azureaiopenai/config/AzureAiOpenAiRecommenderAutoConfiguration.java (80%) delete mode 100644 inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderTraitsEditor.html delete mode 100644 inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt/ChatGptRecommenderTraitsEditor.java rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/ChatGptInteractionPanel.html (100%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/ChatGptInteractionPanel.java (90%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/ChatGptRecommender.java (72%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/ChatGptRecommenderFactory.java (91%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/ChatGptRecommenderTraits.java (55%) create mode 100644 inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/chatgpt/ChatGptRecommenderTraitsEditor.java rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/ChatGptResponseFormatSelect.java (90%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatCompletionChoice.java (95%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatCompletionMessage.java (95%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatCompletionRequest.java (95%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatCompletionResponse.java (95%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatGptClient.java (92%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatGptClientImpl.java (98%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatGptModel.java (96%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ChatGptModelResponse.java (94%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ListModelsRequest.java (95%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ResponseFormat.java (95%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/client/ResponseFormatType.java (93%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/config/ChatGptRecommenderAutoConfiguration.java (81%) rename inception/inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/chatgpt/wicket-package.properties (100%) create mode 100644 inception/inception-imls-llm-support/marker-wicket-module rename inception/{inception-imls-support-llm => inception-imls-llm-support}/pom.xml (94%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/AnnotationWrapper.java (97%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/CasWrapper.java (95%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/JinjaPromptRenderer.java (95%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/PerAnnotationContextGenerator.java (96%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/PerDocumentContextGenerator.java (95%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/PerSentenceContextGenerator.java (96%) rename inception/{inception-imls-chatgpt/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/chatgpt => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/prompt}/Preset.java (77%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/PromptContext.java (95%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/PromptContextGenerator.java (96%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/PromptingMode.java (95%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/prompt/PromptingModeSelect.java (95%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/ExtractionMode.java (94%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/ExtractionModeSelect.java (96%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/MentionsFromJsonExtractor.java (98%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/MentionsSample.java (95%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/ResponseAsLabelExtractor.java (93%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/ResponseExtractor.java (90%) create mode 100644 inception/inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/response/ResponseFormat.java rename inception/{inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaResponseFormatSelect.java => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/response/ResponseFormatSelect.java} (73%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits}/LlmRecommenderImplBase.java (84%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits}/LlmRecommenderTraits.java (67%) rename inception/{inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTraitsEditor.html => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits/LlmRecommenderTraitsEditor_ImplBase.html} (91%) rename inception/{inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/OllamaRecommenderTraitsEditor.java => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits/LlmRecommenderTraitsEditor_ImplBase.java} (77%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits}/Option.java (94%) rename inception/{inception-imls-support-llm/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/option => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support/traits}/OptionSetting.java (95%) rename inception/{inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai => inception-imls-llm-support/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm}/wicket-package.properties (97%) rename inception/{inception-imls-support-llm/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/MentionsFromJsonExtractorTest.java (96%) rename inception/{inception-imls-support-llm/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm => inception-imls-llm-support/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/support}/response/MentionsSampleTest.java (95%) rename inception/{inception-imls-support-llm => inception-imls-llm-support}/src/test/java/de/tudarmstadt/ukp/inception/recommendation/imls/support/llm/prompt/CasWrapperTest.java (96%) rename inception/{inception-imls-support-llm => inception-imls-llm-support}/src/test/resources/log4j2-test.xml (100%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/OllamaInteractionPanel.html (100%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/OllamaInteractionPanel.java (88%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/OllamaRecommender.java (72%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/OllamaRecommenderFactory.java (95%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/OllamaRecommenderTraits.java (51%) create mode 100644 inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/llm/ollama/OllamaRecommenderTraitsEditor.java rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/client/OllamaClient.java (92%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/client/OllamaClientImpl.java (97%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/client/OllamaGenerateRequest.java (97%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/client/OllamaGenerateResponse.java (97%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/client/OllamaGenerateResponseFormat.java (92%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/client/OllamaModel.java (96%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/client/OllamaTagsResponse.java (94%) rename inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/{ => llm}/ollama/config/OllamaRecommenderAutoConfiguration.java (81%) delete mode 100644 inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/Preset.java delete mode 100644 inception/inception-imls-ollama/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/ollama/wicket-package.properties diff --git a/inception/inception-bom/pom.xml b/inception/inception-bom/pom.xml index db2be07d80d..b4e99937569 100644 --- a/inception/inception-bom/pom.xml +++ b/inception/inception-bom/pom.xml @@ -294,7 +294,7 @@ de.tudarmstadt.ukp.inception.app - inception-imls-support-llm + inception-imls-llm-support 35.0-SNAPSHOT diff --git a/inception/inception-imls-azureai-openai/pom.xml b/inception/inception-imls-azureai-openai/pom.xml index bc0855ac86f..83260af6ed7 100644 --- a/inception/inception-imls-azureai-openai/pom.xml +++ b/inception/inception-imls-azureai-openai/pom.xml @@ -29,7 +29,7 @@ de.tudarmstadt.ukp.inception.app - inception-imls-support-llm + inception-imls-llm-support de.tudarmstadt.ukp.inception.app diff --git a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraitsEditor.html b/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraitsEditor.html deleted file mode 100644 index eca569803ad..00000000000 --- a/inception/inception-imls-azureai-openai/src/main/java/de/tudarmstadt/ukp/inception/recommendation/imls/azureaiopenai/AzureAiOpenAiRecommenderTraitsEditor.html +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - -
- -
- -
-
-
- -
-
-
-
-
-
-
- - -
-
-
-
-
-
- -
-
-
- -
- -
-
-
-