From 89e2b07179595c8ff928becea042ccad5494d559 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sat, 2 Nov 2024 21:43:54 +0100 Subject: [PATCH 1/2] #5119 - Tagset in named entity example data does not match data - Made the tagset consistent with the example data - Added another unannotated document to the example data --- .../pom.xml | 28 +++++ .../EntityAnnotationProjectInitializer.java | 31 ++++-- ...amedEntitySampleDataTagSetInitializer.java | 82 ++++++++++++++ ...gProjectInitializersAutoConfiguration.java | 8 ++ ...ict-of-downtown-woodinville-washington.txt | 100 ++++++++++++++++++ .../neannotation/tagsets/foodista.json | 31 ++++++ 6 files changed, 272 insertions(+), 8 deletions(-) create mode 100644 inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/NamedEntitySampleDataTagSetInitializer.java create mode 100644 inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/data/foodista_blog_2024_09_18_the-schoolhouse-district-of-downtown-woodinville-washington.txt create mode 100644 inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/tagsets/foodista.json diff --git a/inception/inception-project-initializers-wikidatalinking/pom.xml b/inception/inception-project-initializers-wikidatalinking/pom.xml index 718f0d2a4bc..05a2ee716d0 100644 --- a/inception/inception-project-initializers-wikidatalinking/pom.xml +++ b/inception/inception-project-initializers-wikidatalinking/pom.xml @@ -28,58 +28,82 @@ de.tudarmstadt.ukp.inception.app inception-support + ${project.version} de.tudarmstadt.ukp.inception.app inception-model + ${project.version} de.tudarmstadt.ukp.inception.app inception-project-api + ${project.version} de.tudarmstadt.ukp.inception.app inception-schema-api + ${project.version} de.tudarmstadt.ukp.inception.app inception-project-initializers + ${project.version} de.tudarmstadt.ukp.inception.app inception-recommendation + ${project.version} de.tudarmstadt.ukp.inception.app inception-imls-stringmatch + ${project.version} de.tudarmstadt.ukp.inception.app inception-imls-opennlp + ${project.version} de.tudarmstadt.ukp.inception.app inception-kb + ${project.version} de.tudarmstadt.ukp.inception.app inception-ui-kb + ${project.version} de.tudarmstadt.ukp.inception.app inception-recommendation-api + ${project.version} de.tudarmstadt.ukp.inception.app inception-io-json + ${project.version} de.tudarmstadt.ukp.inception.app inception-security + ${project.version} de.tudarmstadt.ukp.inception.app inception-documents-api + ${project.version} + + + de.tudarmstadt.ukp.inception.app + inception-export + ${project.version} + + + de.tudarmstadt.ukp.inception.app + inception-io-text + ${project.version} @@ -97,6 +121,10 @@ org.springframework spring-core + + org.springframework + spring-beans + org.springframework spring-context diff --git a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java index 28ef09a40ab..c05bd48d09a 100644 --- a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java +++ b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/EntityAnnotationProjectInitializer.java @@ -34,6 +34,7 @@ import de.tudarmstadt.ukp.clarin.webanno.project.initializers.NamedEntityLayerInitializer; import de.tudarmstadt.ukp.clarin.webanno.project.initializers.QuickProjectInitializer; import de.tudarmstadt.ukp.clarin.webanno.security.UserDao; +import de.tudarmstadt.ukp.clarin.webanno.text.LineOrientedTextFormatSupport; import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; import de.tudarmstadt.ukp.inception.documents.api.DocumentService; import de.tudarmstadt.ukp.inception.io.jsoncas.UimaJsonCasFormatSupport; @@ -100,6 +101,7 @@ public List> getDependencies() { var dependencies = new ArrayList>(); dependencies.add(NamedEntityLayerInitializer.class); + dependencies.add(NamedEntitySampleDataTagSetInitializer.class); if (isStringRecommenderAvailable()) { dependencies.add(NamedEntityStringRecommenderInitializer.class); @@ -130,9 +132,17 @@ public void configure(ProjectInitializationRequest aRequest) throws IOException var project = aRequest.getProject(); project.setName(userService.getCurrentUsername() + " - New entity annotation project"); + var tagset = annotationService + .getTagSet(NamedEntitySampleDataTagSetInitializer.TAG_SET_NAME, project); + var layer = annotationService.findLayer(project, NamedEntity.class.getName()); - var valueFeature = annotationService.getFeature(NamedEntity._FeatName_identifier, layer); - valueFeature.setEnabled(false); + var identifierFeature = annotationService.getFeature(NamedEntity._FeatName_identifier, + layer); + identifierFeature.setEnabled(false); + annotationService.createFeature(identifierFeature); + var valueFeature = annotationService.getFeature(NamedEntity._FeatName_value, layer); + valueFeature.setEnabled(true); + valueFeature.setTagset(tagset); annotationService.createFeature(valueFeature); var description = // @@ -154,29 +164,34 @@ public void configure(ProjectInitializationRequest aRequest) throws IOException } if (aRequest.isIncludeSampleData()) { - importExampleDocument(project, + importExampleDocument(project, UimaJsonCasFormatSupport.ID, "foodista_blog_2019_08_13_northern-british-columbia_abbreviated.json"); - importExampleDocument(project, + importExampleDocument(project, UimaJsonCasFormatSupport.ID, "foodista_blog_2019_10_22_lewiston-clarkstons-new-wine-district_abbreviated.json"); + importExampleDocument(project, LineOrientedTextFormatSupport.ID, + "foodista_blog_2024_09_18_the-schoolhouse-district-of-downtown-woodinville-washington.txt"); description += """ The project includes example documents. Open the **Annotation** page from the left sidbar menu to dive right in. + Two of the three documents come pre-annotated. """; } project.setDescription(description); } - private void importExampleDocument(Project aProject, String docName) throws IOException + private void importExampleDocument(Project aProject, String aFormat, String aDocName) + throws IOException { var doc = SourceDocument.builder() // .withProject(aProject) // - .withName(docName) // - .withFormat(UimaJsonCasFormatSupport.ID) // + .withName(aDocName) // + .withFormat(aFormat) // .build(); - try (var is = getClass().getResourceAsStream("data/" + docName)) { + + try (var is = getClass().getResourceAsStream("data/" + aDocName)) { documentService.uploadSourceDocument(is, doc); } catch (UIMAException e) { diff --git a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/NamedEntitySampleDataTagSetInitializer.java b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/NamedEntitySampleDataTagSetInitializer.java new file mode 100644 index 00000000000..e7b691fca3e --- /dev/null +++ b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/neannotation/NamedEntitySampleDataTagSetInitializer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Technische Universität Darmstadt under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The Technische Universität Darmstadt + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package de.tudarmstadt.ukp.inception.project.initializers.neannotation; + +import static de.tudarmstadt.ukp.inception.export.JsonImportUtil.importTagSetFromJson; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; + +import de.tudarmstadt.ukp.clarin.webanno.model.Project; +import de.tudarmstadt.ukp.clarin.webanno.project.initializers.TagSetInitializer; +import de.tudarmstadt.ukp.inception.project.api.ProjectInitializationRequest; +import de.tudarmstadt.ukp.inception.project.api.ProjectInitializer; +import de.tudarmstadt.ukp.inception.project.initializers.wikidatalinking.config.WikiDataLinkingProjectInitializersAutoConfiguration; +import de.tudarmstadt.ukp.inception.schema.api.AnnotationSchemaService; + +/** + *

+ * This class is exposed as a Spring Component via + * {@link WikiDataLinkingProjectInitializersAutoConfiguration#namedEntitySampleDataTagSetInitializer}. + *

+ */ +public class NamedEntitySampleDataTagSetInitializer + implements TagSetInitializer +{ + // Name must match the name in the tag set file loaded by this initializer! + public static final String TAG_SET_NAME = "Named entity example tags"; + + private final AnnotationSchemaService annotationSchemaService; + + @Autowired + public NamedEntitySampleDataTagSetInitializer(AnnotationSchemaService aAnnotationSchemaService) + { + annotationSchemaService = aAnnotationSchemaService; + } + + @Override + public String getName() + { + return TAG_SET_NAME; + } + + @Override + public List> getDependencies() + { + return Collections.emptyList(); + } + + @Override + public boolean alreadyApplied(Project aProject) + { + return annotationSchemaService.existsTagSet(getName(), aProject); + } + + @Override + public void configure(ProjectInitializationRequest aRequest) throws IOException + { + try (var is = getClass().getResourceAsStream("tagsets/foodista.json")) { + var tagset = importTagSetFromJson(aRequest.getProject(), is, annotationSchemaService); + tagset.setName(TAG_SET_NAME); + annotationSchemaService.createTagSet(tagset); + } + } +} diff --git a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java index 1eb3a20fa91..bc05e09b059 100644 --- a/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java +++ b/inception/inception-project-initializers-wikidatalinking/src/main/java/de/tudarmstadt/ukp/inception/project/initializers/wikidatalinking/config/WikiDataLinkingProjectInitializersAutoConfiguration.java @@ -29,6 +29,7 @@ import de.tudarmstadt.ukp.inception.kb.config.KnowledgeBaseProperties; import de.tudarmstadt.ukp.inception.kb.config.KnowledgeBaseServiceAutoConfiguration; import de.tudarmstadt.ukp.inception.project.initializers.neannotation.EntityAnnotationProjectInitializer; +import de.tudarmstadt.ukp.inception.project.initializers.neannotation.NamedEntitySampleDataTagSetInitializer; import de.tudarmstadt.ukp.inception.project.initializers.neannotation.NamedEntitySequenceClassifierRecommenderInitializer; import de.tudarmstadt.ukp.inception.project.initializers.neannotation.NamedEntityStringRecommenderInitializer; import de.tudarmstadt.ukp.inception.project.initializers.wikidatalinking.EntityLinkingProjectInitializer; @@ -102,4 +103,11 @@ public NamedEntitySequenceClassifierRecommenderInitializer namedEntitySequenceCl return new NamedEntitySequenceClassifierRecommenderInitializer(aRecommenderService, aAnnotationService, aRecommenderFactory); } + + @Bean + public NamedEntitySampleDataTagSetInitializer namedEntitySampleDataTagSetInitializer( + AnnotationSchemaService aAnnotationSchemaService) + { + return new NamedEntitySampleDataTagSetInitializer(aAnnotationSchemaService); + } } diff --git a/inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/data/foodista_blog_2024_09_18_the-schoolhouse-district-of-downtown-woodinville-washington.txt b/inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/data/foodista_blog_2024_09_18_the-schoolhouse-district-of-downtown-woodinville-washington.txt new file mode 100644 index 00000000000..a9886401cbc --- /dev/null +++ b/inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/data/foodista_blog_2024_09_18_the-schoolhouse-district-of-downtown-woodinville-washington.txt @@ -0,0 +1,100 @@ +The Schoolhouse District of Downtown Woodinville, Washington + +Michael and Eli... +September 18, 2024 + +Walla Walla Steak Co, Wine Walk Row, & More + +The Schoolhouse District in downtown Woodinville is a great new food and beverage destination. +Schoolhouse No. 23, located in a restored schoolhouse built in 1909, is the district's centerpiece. +The original red brick and wooden structure is also home to Walla Walla Steak Co. +They have retained the exposed brick features and salvaged elements from the original schoolhouse in the interior design, creating a cozy and welcoming atmosphere. +Interestingly, their original restaurant in Walla Walla is in a remodeled old train station. + +The Schoolhouse District in downtown Woodinville is a newer area with various amenities to enjoy, including multiple wine-tasting rooms, exciting dining options, and seasonal events. +Additionally, it is just a short 30-minute drive from Seattle. + + + +Wine Walk Row + +Six tasting rooms are included in Wine Walk Row, including some iconic wineries from Eastern Washington and the Willamette Valley in Oregon. +Barnard Griffin has Rob Griffin as the head winemaker. +Rob has been making excellent wine in Washington since 1977. +He came up from California with a degree in hand from UC Davis. +His daughter Megan Hughes is currently working with him as a winemaker at their winery in Richland, WA. + + + +Another Eastern Washington winery is L'Ecole No #41 which is based in an old Lowden Schoolhouse. +It is very appropriate that they have a tasting room in the Woodinville Schoolhouse District. +L'Ecole is the French word for school, and 41 is a reference to Walla Wala School District, which is No. 41. +This family-owned winery has roots that go back to 1983. +It was the third winery to open in Walla Walla and just the 20th commercial winery in the state of Washington. + +Valdemar Estates is another great Walla Walla winery with vineyards in Eastern Washington. +Jesús Martínez Bujanda and his sister Ana head up the winery, representing the fifth generation of their family in the winery business. +The original family winery is in the famous Rioja area of Spain, where they grew up. +They bring a touch of Spain to Woodinville with great wine and Pintxo - their lively tapas, wine, and cocktail bar. + + + +The Wine Walk Row includes another Eastern Washington winery, Mullan Road Cellars. +Established in 2012 by Dennis Cakebread – Mullan Road was the result of his search for new winemaking adventures outside of the Napa area. +In the tasting room, you can sample both Mullan Road and Cakebread wines. + +Montinore Estate, a winery in the Willamette Valley in Oregon, brings a multi-faceted approach to growing grapes. +They use the principles of biodynamic farming, which results in one of the smallest carbon footprints of any agricultural practice. +And finally, if you are feeling really festive, you can stop in at McQueen Champaign Bar for a flight of champagne, prosecco, or sparkling rose. + +Cuisine + + + +Walla Walla Steak Co and sister-company Crossbuck Brewing have Dan Thiessen at the helm as managing partner. +He brings over 25 years of culinary experience, including the Pacific Northwest and Switzerland. +Along with this culinary experience, Dan is a 5th generation farmer and owns Thiessen Ranch in Walla Walla. +Dan and his wife Melissa run the farm and provide both restaurants with beef and other farm products. +Along with the Thiessen Ranch Burger on the Crossbuck Brewing Taproom menu, you will find wine from his Wagon Ranch Vineyard. + +Ballard Pizza is another restaurant in the Schoolhouse District. +The Woodinville location is the newest among their three other Seattle locations. +If you go to a Mariners or Seahawks game, you will notice how popular their pizzas are – evidenced by the long line of happy customers waiting for their delicious pies! + +Winemakers Dinner + +Walla Walla Steak Co. hosts winemakers' dinners from time to time, and we were lucky enough to have an excellent one this summer. +We enjoyed the six-course dinner, with each course paired with different wines from the tasting rooms of Wine Walk Row. + +The first course was champagne paired with local oysters from Samish Bay. +The mignonette sauce, with its delicious vinegary and peppery flavors, certainly enhanced the oysters. +Refreshing and light can best describe the next course, which was melon wrapped in burrata cheese and dressed in a lovely elderflower syrup; it was wonderfully paired with Montinore Estate White Pinot Noir. +The white pinot has minimal skin contact during fermentation; thus, it is white and not red. +It is a bit unusual, but highly recommended! + +The next course was Smoked Chicken Roulade wrapped around tangy Roquefort cheese and a sweet blackberry sauce. +The Syrah from Valdemar Estate perfectly balanced this complex and tasty dish. + + + +The following courses included King Salmon and New York Strip, each excellently paired with wine from the local tasting rooms. +The Chef's Dessert Trio was an excellent finish, along with the port-style wine Rapport from Barnard Griffin Winery. +The outstanding dessert wine was elegant and fruit-forward with slight chocolaty notes. +It was the perfect conclusion to a delightful dining experience. + +Public Events + +There are many upcoming events scheduled, some that take place indoors and many that take advantage of the lovely outdoor spaces of the Schoolhouse District. +You will also find the YMCA and the YMCA Early Education Center at the Carol Edwards Center. +The YMCA offers a wide-ranging list of activities, such as pickleball and basketball courts, indoor and outdoor rock climbing, fitness classes, cooking, art, music, and drama classes, and much more. +The Early Education Center provides childcare and learning programs for kids aged 1 to 6, along with before and after-school programs for older children. +Incidentally, Carol Edwards is known as the "Mother of Woodinville". +She established the local paper in 1976, organized the first All Fool's Day Parade, and founded the Woodinville Community Band. +She is often credited with empowering the community to become the vibrant cultural center that it is. +She died of ALS in 2007 at the age of 65, just hours after the Carol Edwards Center was named in her honor. + +Editorial disclosure: food, and beverages generously provided. + +Source: Foodista.com - The Cooking Encyclopedia Everyone Can Edit +License: http://creativecommons.org/licenses/by/3.0/ +URL: https://www.foodista.com/blog/2024/09/18/the-schoolhouse-district-of-downtown-woodinville-washington diff --git a/inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/tagsets/foodista.json b/inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/tagsets/foodista.json new file mode 100644 index 00000000000..5f4f4cdec9b --- /dev/null +++ b/inception/inception-project-initializers-wikidatalinking/src/main/resources/de/tudarmstadt/ukp/inception/project/initializers/neannotation/tagsets/foodista.json @@ -0,0 +1,31 @@ +{ + "name" : "Named entity - value", + "description" : null, + "language" : null, + "tags" : [ { + "tag_name" : "Date", + "tag_description" : null + }, { + "tag_name" : "Event", + "tag_description" : null + }, { + "tag_name" : "Location", + "tag_description" : null + }, { + "tag_name" : "Organization", + "tag_description" : null + }, { + "tag_name" : "Person", + "tag_description" : null + }, { + "tag_name" : "Product", + "tag_description" : null + }, { + "tag_name" : "Product Category", + "tag_description" : null + }, { + "tag_name" : "Route", + "tag_description" : null + } ], + "create_tag" : false +} \ No newline at end of file From 3cb114b5737c4430ad76974ae4f438717db46e53 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sat, 2 Nov 2024 22:06:48 +0100 Subject: [PATCH 2/2] #5121 - Improve recommender confusion matrix - Add better coloring and better explanations --- .../api/evaluation/ConfusionMatrix.java | 8 ++-- .../api/evaluation/EvaluationResult.java | 21 +++++---- .../sidebar/RecommenderStatusDetailPanel.html | 23 ++++------ .../sidebar/RecommenderStatusDetailPanel.java | 46 ++++++++++++------- 4 files changed, 56 insertions(+), 42 deletions(-) diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/ConfusionMatrix.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/ConfusionMatrix.java index 76882ab7157..2eb3530d3a6 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/ConfusionMatrix.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/ConfusionMatrix.java @@ -112,7 +112,7 @@ public void addMatrix(ConfusionMatrix aMatrix) @Override public String toString() { - StringBuilder matrixStr = new StringBuilder(); + var matrixStr = new StringBuilder(); // header matrixStr.append("Gold\\Predicted\n\t"); labels.forEach(l -> { @@ -120,8 +120,9 @@ public String toString() matrixStr.append(" "); }); matrixStr.append("\n"); + // table - for (String goldLabel : labels) { + for (var goldLabel : labels) { matrixStr.append(goldLabel); matrixStr.append("\t| "); for (String predictedLabel : labels) { @@ -131,6 +132,7 @@ public String toString() } matrixStr.append("\n"); } + return matrixStr.toString(); } @@ -164,7 +166,7 @@ public boolean equals(Object aObj) return false; } - ConfMatrixKey aKey = (ConfMatrixKey) aObj; + var aKey = (ConfMatrixKey) aObj; return predictedLabel.equals(aKey.getPredictedLabel()) && goldLabel.equals(aKey.getGoldLabel()); } diff --git a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/EvaluationResult.java b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/EvaluationResult.java index 9326a51406c..21d9ba7079b 100644 --- a/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/EvaluationResult.java +++ b/inception/inception-recommendation-api/src/main/java/de/tudarmstadt/ukp/inception/recommendation/api/evaluation/EvaluationResult.java @@ -103,9 +103,10 @@ public int getNumOfLabels() if (ignoreLabels.isEmpty()) { return labels.size(); } - else { - return Math.toIntExact(labels.stream().filter(l -> !ignoreLabels.contains(l)).count()); - } + + return Math.toIntExact(labels.stream() // + .filter(l -> !ignoreLabels.contains(l)) // + .count()); } /** @@ -115,16 +116,16 @@ public int getNumOfLabels() */ public double computeAccuracyScore() { - double tp = 0.0; - double ignoreLabelAsGold = 0; - for (String label : confusionMatrix.getLabels()) { + var tp = 0.0d; + var ignoreLabelAsGold = 0.0d; + for (var label : confusionMatrix.getLabels()) { if (!ignoreLabels.contains(label)) { tp += confusionMatrix.getEntryCount(label, label); } ignoreLabelAsGold += countIgnoreLabelsAsGold(label); } double total = confusionMatrix.getTotal() - ignoreLabelAsGold; - return (total > 0) ? tp / total : 0.0; + return (total > 0) ? tp / total : 0.0d; } /** @@ -133,8 +134,8 @@ public double computeAccuracyScore() */ private double countIgnoreLabelsAsGold(String label) { - double ignoreLabelAsGold = 0.0; - for (String ignoreLabel : ignoreLabels) { + var ignoreLabelAsGold = 0.0d; + for (var ignoreLabel : ignoreLabels) { ignoreLabelAsGold += confusionMatrix.getEntryCount(label, ignoreLabel); } return ignoreLabelAsGold; @@ -185,7 +186,7 @@ private double calcMetricAverage(ToDoubleBiFunction countFunctio tp = confusionMatrix.getEntryCount(label, label); } double numIsLabel = 0.0; - for (String predictedLabel : labels) { + for (var predictedLabel : labels) { numIsLabel += countFunction.applyAsDouble(label, predictedLabel); } metric += calcClassMetric(label, tp, numIsLabel); diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.html b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.html index 1b62ba80c56..6203c7065d4 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.html +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.html @@ -20,15 +20,11 @@
Confusion Matrix
- +
- Expected label in rows, predicted label in columns. + Gold labels in rows, predicted labels in columns.
@@ -41,22 +37,23 @@
Confusion Matrix
- + Evaluation is performed on a single train/test split. + The confusion matrix only covers the data in the test set. + A full cross-validation over all data is not performed. +
+
+ @ - + -level
- +
- diff --git a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.java b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.java index 84cdc669812..c7db90098b7 100644 --- a/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.java +++ b/inception/inception-recommendation/src/main/java/de/tudarmstadt/ukp/inception/recommendation/sidebar/RecommenderStatusDetailPanel.java @@ -21,11 +21,12 @@ import static java.util.stream.Collectors.toList; import static java.util.stream.Stream.concat; -import java.util.List; import java.util.stream.Stream; import org.apache.wicket.AttributeModifier; +import org.apache.wicket.ajax.AjaxRequestTarget; import org.apache.wicket.behavior.AttributeAppender; +import org.apache.wicket.extensions.ajax.markup.html.modal.ModalDialog; import org.apache.wicket.markup.html.WebMarkupContainer; import org.apache.wicket.markup.html.basic.Label; import org.apache.wicket.markup.html.panel.Fragment; @@ -37,6 +38,7 @@ import de.tudarmstadt.ukp.inception.recommendation.api.evaluation.ConfusionMatrix; import de.tudarmstadt.ukp.inception.recommendation.api.evaluation.EvaluationResult; +import de.tudarmstadt.ukp.inception.support.lambda.LambdaAjaxLink; import de.tudarmstadt.ukp.inception.support.wicket.DefaultRefreshingView; public class RecommenderStatusDetailPanel @@ -50,17 +52,17 @@ public RecommenderStatusDetailPanel(String aId, IModel aModel) { super(aId, aModel); - IModel confusionMatrixData = aModel - .map(EvaluationResult::getConfusionMatrix); + var confusionMatrixData = aModel.map(EvaluationResult::getConfusionMatrix); // Prepend a dummy element for the header columns into the axes - IModel> axes = confusionMatrixData.map(ConfusionMatrix::getLabels) + var axes = confusionMatrixData.map(ConfusionMatrix::getLabels) .map(label -> concat(Stream.of(""), label.stream()).collect(toList())); - add(new Label("totalDatapoints", confusionMatrixData.map(ConfusionMatrix::getTotal))); - add(new Label("datapointUnit", confusionMatrixData.map(ConfusionMatrix::getUnit))); - add(new Label("ignoredLabels", aModel.map(EvaluationResult::getIgnoreLabels) + queue(new Label("totalDatapoints", confusionMatrixData.map(ConfusionMatrix::getTotal))); + queue(new Label("datapointUnit", confusionMatrixData.map(ConfusionMatrix::getUnit))); + queue(new Label("ignoredLabels", aModel.map(EvaluationResult::getIgnoreLabels) .map(labels -> labels.stream().collect(joining(", "))))); + queue(new LambdaAjaxLink("closeDialog", this::actionCloseDialog)); rows = new DefaultRefreshingView("rows", axes) { @@ -124,19 +126,26 @@ else if (aRowItem.getIndex() != 0 && aCellItem.getIndex() == 0) { private void renderCell(IModel aModel, Item aRowItem, Item aCellItem, Fragment aCell) { - String predicted = aCellItem.getModelObject(); - String gold = aRowItem.getModelObject(); - - WebMarkupContainer cellContent = new WebMarkupContainer("cellContent"); - // if (aCellItem.getIndex() == aRowItem.getIndex()) { - // cellContent.add(AttributeAppender.append("class", "bg-success")); - // } + var predicted = aCellItem.getModelObject(); + var gold = aRowItem.getModelObject(); + var count = aModel.map(m -> m.getEntryCount(predicted, gold)); + + var cellContent = new WebMarkupContainer("cellContent"); + if (aCellItem.getIndex() == aRowItem.getIndex()) { + if (!getModel().map(EvaluationResult::getIgnoreLabels) + .map(ignLabels -> ignLabels.contains(gold)).getObject()) { + cellContent.add(AttributeAppender.append("class", "text-bg-success")); + } + } + else if (count.map(v -> v > 0).getObject()) { + cellContent.add(AttributeAppender.append("class", "text-bg-danger")); + } aCell.queue(cellContent); - Label label = new Label("label", aModel.map(m -> m.getEntryCount(predicted, gold))); + var label = new Label("label", count); label.add(AttributeModifier.append("class", getModel() // .map(EvaluationResult::getIgnoreLabels) // - .map(ignLabels -> ignLabels.contains(gold)) // + .map(ignLabels -> ignLabels.contains(gold) && predicted.equals(gold)) // .map(ignored -> ignored ? "text-muted" : ""))); aCell.queue(label); } @@ -151,4 +160,9 @@ public EvaluationResult getModelObject() { return (EvaluationResult) getDefaultModelObject(); } + + protected void actionCloseDialog(AjaxRequestTarget aTarget) + { + findParent(ModalDialog.class).close(aTarget); + } }