From f8e7791a73d5986957e038534251e0f48e73cbcc Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sat, 9 Dec 2023 22:36:28 +0100 Subject: [PATCH] #4370 - Download knowledge bases in compressed form - Download immediately without first buffering into a temporary file - GZip exported KBs - Add Turtle to list of supported formats - Removed unused property --- .../support/wicket/PipedStreamResource.java | 19 +++++++++++++++++-- .../ProjectKnowledgeBasePanel.properties | 3 +-- .../local/LocalRepositorySettingsPanel.java | 17 ++++++++++------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/wicket/PipedStreamResource.java b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/wicket/PipedStreamResource.java index 68d7ca63dbb..9e8e0dbd0e9 100644 --- a/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/wicket/PipedStreamResource.java +++ b/inception/inception-support/src/main/java/de/tudarmstadt/ukp/inception/support/wicket/PipedStreamResource.java @@ -17,6 +17,8 @@ */ package de.tudarmstadt.ukp.inception.support.wicket; +import static java.util.Arrays.asList; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -24,6 +26,7 @@ import java.io.PipedOutputStream; import java.io.Serializable; import java.lang.invoke.MethodHandles; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.wicket.util.resource.AbstractResourceStream; @@ -42,11 +45,13 @@ public class PipedStreamResource private final DataSupplier supplier; private PipedInputStream is; - private PipedOutputStream os; + private OutputStream os; + private List filters; - public PipedStreamResource(DataSupplier aSupplier) + public PipedStreamResource(DataSupplier aSupplier, Filter... aFilters) { supplier = aSupplier; + filters = asList(aFilters); } @Override @@ -56,6 +61,9 @@ public InputStream getInputStream() throws ResourceStreamNotFoundException try { os = new PipedOutputStream(is); + for (var filter : filters) { + os = filter.apply(os); + } } catch (IOException e) { throw new ResourceStreamNotFoundException(e); @@ -106,4 +114,11 @@ public interface DataSupplier { void write(OutputStream aOS) throws IOException; } + + @FunctionalInterface + public interface Filter + extends Serializable + { + OutputStream apply(OutputStream aT) throws IOException; + } } diff --git a/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/ProjectKnowledgeBasePanel.properties b/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/ProjectKnowledgeBasePanel.properties index 887da75d385..c15a4c1508b 100644 --- a/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/ProjectKnowledgeBasePanel.properties +++ b/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/ProjectKnowledgeBasePanel.properties @@ -45,8 +45,7 @@ kb.language=Language kb.supportConceptLinking=Supports Concept Linking kb.queryLimit=Result Limit for SPARQL queries -kb.local.fileupload.supported.list=Supported file types: RDF (XML/JSON/Binary), JSON-LD, N-Triples, N-Quads, TriG, TriX -kb.local.fileupload.supported.headerlist: Supported file types: RDF (XML/JSON/Binary), JSON-LD, N-Triples, N-Quads, TriG, TriX +kb.local.fileupload.supported.list=Supported: RDF (XML/JSON/Binary), JSON-LD, N-Triples, N-Quads, TriG, TriX, Turtle (optionally gzipped) kb.wizard.steps.type.language=Language kb.wizard.steps.type.name=Name diff --git a/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/local/LocalRepositorySettingsPanel.java b/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/local/LocalRepositorySettingsPanel.java index 3b6c5d0dcd5..f5a71e4b135 100644 --- a/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/local/LocalRepositorySettingsPanel.java +++ b/inception/inception-ui-kb/src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/local/LocalRepositorySettingsPanel.java @@ -36,6 +36,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; import org.apache.commons.io.FileUtils; import org.apache.wicket.AttributeModifier; @@ -76,7 +77,7 @@ import de.tudarmstadt.ukp.inception.support.io.FileUploadDownloadHelper; import de.tudarmstadt.ukp.inception.support.lambda.LambdaAjaxLink; import de.tudarmstadt.ukp.inception.support.wicket.AjaxDownloadLink; -import de.tudarmstadt.ukp.inception.support.wicket.TempFileResource; +import de.tudarmstadt.ukp.inception.support.wicket.PipedStreamResource; import de.tudarmstadt.ukp.inception.ui.kb.project.AccessSpecificSettingsPanel; import de.tudarmstadt.ukp.inception.ui.kb.project.KnowledgeBaseInfoPanel; import de.tudarmstadt.ukp.inception.ui.kb.project.KnowledgeBaseWrapper; @@ -254,10 +255,10 @@ protected void populateItem(ListItem item) { // creates an appropriately labeled {@link AjaxDownloadLink} which triggers the // download of the contents of the current KB in the given format - String fileExtension = item.getModelObject(); - KnowledgeBase kb = LocalRepositorySettingsPanel.this.getModel().getObject().getKb(); - Model exportFileNameModel = Model.of(kb.getName() + "." + fileExtension); - AjaxDownloadLink exportLink = new AjaxDownloadLink("link", exportFileNameModel, + var fileExtension = item.getModelObject(); + var kb = LocalRepositorySettingsPanel.this.getModel().getObject().getKb(); + var exportFileNameModel = Model.of(kb.getName() + "." + fileExtension + ".gz"); + var exportLink = new AjaxDownloadLink("link", exportFileNameModel, LoadableDetachableModel.of(() -> actionExport(fileExtension))); exportLink.add(new Label("label", new ResourceModel("kb.export." + fileExtension))); item.add(exportLink); @@ -378,8 +379,10 @@ private void actionClear(AjaxRequestTarget aTarget) private IResourceStream actionExport(String rdfFormatFileExt) { - return new TempFileResource((os) -> kbService.exportData(getModel().getObject().getKb(), - getRdfFormatForFileExt(rdfFormatFileExt), os)); + var kb = getModel().getObject().getKb(); + var format = getRdfFormatForFileExt(rdfFormatFileExt); + return new PipedStreamResource((os) -> kbService.exportData(kb, format, os), + GZIPOutputStream::new); } private void actionDownloadKbAndSetIRIs(AjaxRequestTarget aTarget)