Skip to content

Commit

Permalink
Ensures that external resources are not exported by default. (#158)
Browse files Browse the repository at this point in the history 
* Ensures that external resources are not exported by default.
Users must now specify --external=true to export proxied and redirected external resources.
* ensure inputstream is closed
* add test to verify that proxy handling works as well.

Resolves: https://jira.lyrasis.org/browse/FCREPO-3416
  • Loading branch information
@dbernstein
dbernstein authored Feb 3, 2021
1 parent 310466b commit e7aa8bd
Show file tree
Hide file tree
Showing 4 changed files with 99 additions and 12 deletions.
18 changes: 12 additions & 6 deletions src/main/java/org/fcrepo/importexport/exporter/Exporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import java.io.OutputStream;
import java.io.Reader;
import java.net.URI;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -388,7 +389,7 @@ private void exportBinary(final URI uri, final List<URI> describedby, final bool
throws FcrepoOperationFailedException, IOException {

if (!config.isIncludeBinaries()) {
logger.info("Skipping {}", uri);
logger.debug("Skipping: {} -> binaries are not included in this export configuration", uri);
return;
}

Expand All @@ -402,11 +403,16 @@ private void exportBinary(final URI uri, final List<URI> describedby, final bool
final File file = external ? fileForExternalResources(uri, null, null, config.getBaseDirectory()) :
fileForBinary(uri, null, null, config.getBaseDirectory());

logger.info("Exporting binary: {}", uri);
writeResponse(uri, response.getBody(), describedby, file);
writeHeadersFile(response, getHeadersFile(file));
exportLogger.info("export {} to {}", uri, file.getAbsolutePath());
successCount.incrementAndGet();
//only retrieve content of external resources when retrieve external flag is enabled
//otherwise write a zero length file.
try (final InputStream is = external && !config.retrieveExternal() ?
IOUtils.toInputStream("", Charset.defaultCharset()) : response.getBody()) {
logger.info("Exporting binary: {}", uri);
writeResponse(uri, is, describedby, file);
writeHeadersFile(response, getHeadersFile(file));
exportLogger.info("export {} to {}", uri, file.getAbsolutePath());
successCount.incrementAndGet();
}

}

Expand Down
2 changes: 2 additions & 0 deletions src/test/java/org/fcrepo/importexport/exporter/ExporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ public void testExportBag() throws Exception {
bagArgs.setMode("export");
bagArgs.setBaseDirectory(basedir);
bagArgs.setIncludeBinaries(true);
bagArgs.setRetrieveExternal(true);
bagArgs.setPredicates(predicates);
bagArgs.setRdfLanguage("application/ld+json");
bagArgs.setResource(resource3);
Expand Down Expand Up @@ -297,6 +298,7 @@ public void testExportBeyondTheRepositoryBag() throws IOException {
config.setMode("export");
config.setBaseDirectory(exportDirectory);
config.setIncludeBinaries(true);
config.setRetrieveExternal(true);
config.setPredicates(predicates);
config.setRdfLanguage("application/ld+json");
config.setResource(resource3);
Expand Down
71 changes: 71 additions & 0 deletions src/test/java/org/fcrepo/importexport/integration/ExporterIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.slf4j.LoggerFactory.getLogger;

/**
Expand Down Expand Up @@ -176,6 +177,76 @@ public void testExportRetrieveExternal() throws Exception {
assertEquals("content", readFileToString(externalFile, "UTF-8"));
}

@Test
public void testExportRetrieveExternalDisabledWithRedirect() throws Exception {
// Create an external content resource pointing at another repository resource
final URI binaryURI = URI.create(serverAddress + UUID.randomUUID());
createBody(binaryURI, "content", "text/plain");
final Map<String,String> headers = new HashMap<>();
headers.put("Link", "<" + binaryURI.toString() + ">;" +
" rel=\"http://fedora.info/definitions/fcrepo#ExternalContent\"; " +
" handling=\"redirect\"; " +
" type=\"text/plain\"");

createBody(url, new ByteArrayInputStream("".getBytes()), "text/plain", headers);

// Run an export process
final Config config = new Config();
config.setMode("export");
config.setBaseDirectory(TARGET_DIR);
config.setResource(url);
config.setIncludeBinaries(true);
config.setRetrieveExternal(false);
config.setUsername(USERNAME);
config.setPassword(PASSWORD);

final Exporter exporter = new Exporter(config, clientBuilder);
exporter.run();

// Verify
final File externalFile = new File(TARGET_DIR, url.getPath() + EXTERNAL_RESOURCE_EXTENSION);
assertTrue(externalFile.exists());
assertEquals("File length should be 0", 0, externalFile.length());
}

@Test
public void testExportRetrieveExternalDisabledWithProxy() throws Exception {
// Create an external content resource pointing at a local file resource
final File file = File.createTempFile("test",".txt");
file.deleteOnExit();
final String fileStr = "file://" + file.toPath().toAbsolutePath().toString();
logger().debug("External File Resource = {}", fileStr);
final Map<String,String> headers = new HashMap<>();
headers.put("Link", "<" + fileStr + ">;" +
"rel=\"http://fedora.info/definitions/fcrepo#ExternalContent\"; " +
"handling=\"proxy\"; type=\"text/plain\"");
final FcrepoResponse proxiedResourceResponse = createBody(url, new ByteArrayInputStream("".getBytes()),
"text/plain", headers);
assertEquals(SC_CREATED, proxiedResourceResponse.getStatusCode());
// Run an export process
final Config config = new Config();
config.setMode("export");
config.setBaseDirectory(TARGET_DIR);
config.setResource(url);
config.setIncludeBinaries(true);
config.setRetrieveExternal(false);
config.setUsername(USERNAME);
config.setPassword(PASSWORD);

final Exporter exporter = new Exporter(config, clientBuilder);
try {
exporter.run();
} catch(Exception ex) {
logger().error(ex.getMessage(), ex);
fail("Exporter failed unexpectedly: " + ex.getMessage());
}

// Verify
final File externalFile = new File(TARGET_DIR, url.getPath() + EXTERNAL_RESOURCE_EXTENSION);
assertTrue("External file should exist: " + externalFile.getAbsoluteFile(), externalFile.exists());
assertEquals("File length should be 0", 0, externalFile.length());
}

@Test
public void testExportCustomPredicate() throws Exception {
final String[] predicates = new String[]{ "http://example.org/custom" };
Expand Down
20 changes: 14 additions & 6 deletions src/test/java/org/fcrepo/importexport/integration/RoundtripIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ public void testRoundtripWithIgnoreVersions() throws Exception {
final FcrepoResponse memento = post(timemapURI);
assertEquals(SC_CREATED, memento.getStatusCode());
final URI mementoURI = memento.getLocation();
roundtrip(uri, new ArrayList<URI>(), true, true, false);
roundtrip(uri, new ArrayList<URI>(), true, true, false, false);
//timemap will be there because it is created automatically
assertEquals(SC_OK, get(timemapURI).getStatusCode());
//the memento however should not have been imported.
Expand Down Expand Up @@ -390,7 +390,7 @@ public void testRoundtripExcludedBinaries() throws Exception {
assertTrue(exists(res1));
assertTrue(exists(file1));

final Config config = roundtrip(res1, Collections.singletonList(file1), true, false, true);
final Config config = roundtrip(res1, Collections.singletonList(file1), true, false, true, false);

// verify that files exist and contain expected content
final File exportDir = config.getBaseDirectory();
Expand Down Expand Up @@ -498,7 +498,7 @@ public void testRoundtripExternalProxy() throws Exception {

final String contentLength = get(file1).getHeaderValue("Content-Length");
assertEquals("Unexpected Content-Length value", contentLength, binaryFileLength + "");
final Config config = roundtrip(URI.create(baseURI), true);
final Config config = roundtrip(URI.create(baseURI), true,true);

// verify that files exist and contain expected content
final File exportDir = config.getBaseDirectory();
Expand Down Expand Up @@ -648,12 +648,19 @@ private Literal longLiteral(final String longString) {
return createTypedLiteral(longString, XSDlong);
}

private Config roundtrip(final URI uri, final boolean reset) throws FcrepoOperationFailedException {
return roundtrip(uri, new ArrayList<URI>(), true, true, true);
private Config roundtrip(final URI uri, final boolean reset)
throws FcrepoOperationFailedException {
return roundtrip(uri, new ArrayList<URI>(), true, true, true, false);
}

private Config roundtrip(final URI uri, final boolean reset, final boolean retrieveExternal)
throws FcrepoOperationFailedException {
return roundtrip(uri, new ArrayList<URI>(), true, true, true, retrieveExternal);
}

private Config roundtrip(final URI uri, final List<URI> relatedResources,
final boolean reset, final boolean includeBinary, final boolean includeVersions)
final boolean reset, final boolean includeBinary, final boolean includeVersions,
final boolean retrieveExternal)
throws FcrepoOperationFailedException {
// export resources
final Config config = new Config();
Expand All @@ -662,6 +669,7 @@ private Config roundtrip(final URI uri, final List<URI> relatedResources,
config.setIncludeBinaries(includeBinary);
config.setIncludeMembership(true);
config.setResource(uri);
config.setRetrieveExternal(retrieveExternal);
config.setPredicates(new String[]{ CONTAINS.toString() });
config.setRdfExtension(DEFAULT_RDF_EXT);
config.setRdfLanguage(DEFAULT_RDF_LANG);
Expand Down

0 comments on commit e7aa8bd

Please sign in to comment.