From 27ba90c83d4ebea1a26ab80a88ea035c69e1e7c4 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Thu, 21 Feb 2019 12:57:56 -0500 Subject: [PATCH] FCREPO-2459 Import of versions (#109) * Initial versioned importer * Added chronological resource iterator * Moved json ld construction for test to a utility * Added Subject Mapper for versioned resources * Excluding versioned resources when generating iterator of import resources if versions are being excluded * Switched to release version of fcrepo-java-client * Added constant for Version rdf type * Refactored import resource factory to shift most of the file discovery over to the iterator * Changed scope of some components to allow for extension * Refactored VersionImporter to use a chronological iterator for resources. Added partial unit test. * Switched to an event perspective for tracking import of resources and versions. Incorporated timestamp detection of unmodified resources to prevent unnecessary re-import of resources * Refactored ImportEvent classes into separate files. Removed dead code * Getting more of the tests working * Addressing ordering issues when parent changes * Fixed checkstyle issues * Refactored iterator to default to sorting by created except in the case of versions after the original, which are sorted by last modified * Got CorruptedBinary IT working * Replaced Importer with VersionImporter * Added back in special santizing of membership relations * Added in verifyBag method for retention purposes * Pulled in some changes from master to get the last IT working * Incorporated containerBuilder bag functionality into import of descriptions * Added round trip IT for versions * Implemented cleanup of resources that are added in a version but later removed * Close iterators and correct tests related to the event iterator * Cleanup tombstones of objects which are deleted in past versions of resources while recreating the versions. Delete unused variables. Update deleteTombstone method to figure out the tombstone uri for the given rescUri if no tombstone uri was given (previously it attempted to get the tombstone uri of the parent, but this was not used). Add integration test to verify import of versions with deleted children. * Cleaned up unused code related to using digests during binary ingest Resolves: https://jira.duraspace.org/browse/FCREPO-2459 --- .../common/BinaryImportException.java | 47 ++ .../importexport/common/FcrepoConstants.java | 6 + .../importexport/common/ModelUtils.java | 79 +++ .../common/ResourceGoneRuntimeException.java | 58 ++ .../importexport/common/TransferProcess.java | 2 +- .../common/URITranslationUtil.java | 120 ++++ .../ChronologicalImportEventIterator.java | 304 +++++++++ .../importexport/importer/ImportDeletion.java | 43 ++ .../importexport/importer/ImportEvent.java | 111 ++++ .../importexport/importer/ImportResource.java | 153 +++++ .../importexport/importer/ImportVersion.java | 69 ++ .../importexport/importer/Importer.java | 593 ++++++++---------- .../importer/SubjectMappingStreamRDF.java | 13 +- .../VersionDiffDeletionGenerator.java | 208 ++++++ .../VersionSubjectMappingStreamRDF.java | 58 ++ .../exporter/ExportVersionsTest.java | 47 +- .../ChronologicalImportEventIteratorTest.java | 267 ++++++++ .../importer/ImportVersionsTest.java | 198 ++++++ .../importexport/importer/ImporterTest.java | 28 +- .../VersionDiffDeletionGeneratorTest.java | 123 ++++ .../VersionSubjectMappingStreamRDFTest.java | 172 +++++ .../integration/AbstractResourceIT.java | 9 + .../importexport/integration/ExporterIT.java | 9 +- .../importexport/integration/ImporterIT.java | 17 +- .../importexport/integration/RoundtripIT.java | 141 ++++- .../test/util/JsonLdResponse.java | 78 +++ .../test/util/ResponseMocker.java | 102 ++- .../sample/recent_parent/rest/con1.jsonld | 1 + .../recent_parent/rest/con1/con2.jsonld | 1 + .../rest/con1/fcr%3Aversions.jsonld | 1 + .../rest/con1/fcr%3Aversions/v0.jsonld | 1 + .../rest/con1/fcr%3Aversions/v0/con2.jsonld | 1 + .../sample/versioned/fcrepo/rest/con1.jsonld | 1 + .../versioned/fcrepo/rest/con1/bin1.binary | Bin 0 -> 32 bytes .../rest/con1/bin1/fcr%3Ametadata.jsonld | 1 + .../fcrepo/rest/con1/fcr%3Aversions.jsonld | 1 + .../rest/con1/fcr%3Aversions/version_1.jsonld | 1 + .../con1/fcr%3Aversions/version_1/bin1.binary | 1 + .../version_1/bin1/fcr%3Ametadata.jsonld | 1 + .../fcr%3Aversions/version_original.jsonld | 1 + .../version_original/con2.jsonld | 1 + .../binary_removed/fcrepo/rest/con1.jsonld | 1 + .../fcrepo/rest/con1/fcr%3Aversions.jsonld | 1 + .../rest/con1/fcr%3Aversions/version_1.jsonld | 1 + .../con1/fcr%3Aversions/version_1/bin1.binary | 1 + .../version_1/bin1/fcr%3Ametadata.jsonld | 1 + .../container_added/rest/con1/child1.jsonld | 1 + .../rest/con1/fcr%3Aversions.jsonld | 1 + .../rest/con1/fcr%3Aversions/version_1.jsonld | 1 + .../fcr%3Aversions/version_1/child1.jsonld | 1 + .../fcr%3Aversions/version_original.jsonld | 37 ++ .../container_added/rest/v_con1.jsonld | 1 + .../rest/con1/child1.jsonld | 1 + .../rest/con1/fcr%3Aversions.jsonld | 1 + .../rest/con1/fcr%3Aversions/version_1.jsonld | 1 + .../fcr%3Aversions/version_original.jsonld | 1 + .../version_original/child1.jsonld | 1 + .../container_restored/rest/v_con1.jsonld | 1 + .../removed_in_head/fcrepo/rest/con1.jsonld | 1 + .../fcrepo/rest/con1/fcr%3Aversions.jsonld | 1 + .../fcr%3Aversions/version_original.jsonld | 1 + .../version_original/con2.jsonld | 1 + .../versioning/unmodified/rest/v_con1.jsonld | 1 + .../unmodified/rest/v_con1/child1.jsonld | 1 + .../rest/v_con1/fcr%3Aversions.jsonld | 1 + .../fcr%3Aversions/version_original.jsonld | 1 + .../version_original/child1.jsonld | 1 + .../fcr%3Aversions/version_unchanged.jsonld | 1 + .../version_unchanged/child1.jsonld | 1 + 69 files changed, 2691 insertions(+), 441 deletions(-) create mode 100644 src/main/java/org/fcrepo/importexport/common/BinaryImportException.java create mode 100644 src/main/java/org/fcrepo/importexport/common/ModelUtils.java create mode 100644 src/main/java/org/fcrepo/importexport/common/ResourceGoneRuntimeException.java create mode 100644 src/main/java/org/fcrepo/importexport/common/URITranslationUtil.java create mode 100644 src/main/java/org/fcrepo/importexport/importer/ChronologicalImportEventIterator.java create mode 100644 src/main/java/org/fcrepo/importexport/importer/ImportDeletion.java create mode 100644 src/main/java/org/fcrepo/importexport/importer/ImportEvent.java create mode 100644 src/main/java/org/fcrepo/importexport/importer/ImportResource.java create mode 100644 src/main/java/org/fcrepo/importexport/importer/ImportVersion.java create mode 100644 src/main/java/org/fcrepo/importexport/importer/VersionDiffDeletionGenerator.java create mode 100644 src/main/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDF.java create mode 100644 src/test/java/org/fcrepo/importexport/importer/ChronologicalImportEventIteratorTest.java create mode 100644 src/test/java/org/fcrepo/importexport/importer/ImportVersionsTest.java create mode 100644 src/test/java/org/fcrepo/importexport/importer/VersionDiffDeletionGeneratorTest.java create mode 100644 src/test/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDFTest.java create mode 100644 src/test/java/org/fcrepo/importexport/test/util/JsonLdResponse.java create mode 100644 src/test/resources/sample/recent_parent/rest/con1.jsonld create mode 100644 src/test/resources/sample/recent_parent/rest/con1/con2.jsonld create mode 100644 src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions.jsonld create mode 100644 src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0.jsonld create mode 100644 src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0/con2.jsonld create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1.jsonld create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/bin1.binary create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/bin1/fcr%3Ametadata.jsonld create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/fcr%3Aversions.jsonld create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/fcr%3Aversions/version_1.jsonld create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/fcr%3Aversions/version_1/bin1.binary create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/fcr%3Aversions/version_1/bin1/fcr%3Ametadata.jsonld create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/fcr%3Aversions/version_original.jsonld create mode 100644 src/test/resources/sample/versioned/fcrepo/rest/con1/fcr%3Aversions/version_original/con2.jsonld create mode 100644 src/test/resources/sample/versioning/binary_removed/fcrepo/rest/con1.jsonld create mode 100644 src/test/resources/sample/versioning/binary_removed/fcrepo/rest/con1/fcr%3Aversions.jsonld create mode 100644 src/test/resources/sample/versioning/binary_removed/fcrepo/rest/con1/fcr%3Aversions/version_1.jsonld create mode 100644 src/test/resources/sample/versioning/binary_removed/fcrepo/rest/con1/fcr%3Aversions/version_1/bin1.binary create mode 100644 src/test/resources/sample/versioning/binary_removed/fcrepo/rest/con1/fcr%3Aversions/version_1/bin1/fcr%3Ametadata.jsonld create mode 100644 src/test/resources/sample/versioning/container_added/rest/con1/child1.jsonld create mode 100644 src/test/resources/sample/versioning/container_added/rest/con1/fcr%3Aversions.jsonld create mode 100644 src/test/resources/sample/versioning/container_added/rest/con1/fcr%3Aversions/version_1.jsonld create mode 100644 src/test/resources/sample/versioning/container_added/rest/con1/fcr%3Aversions/version_1/child1.jsonld create mode 100644 src/test/resources/sample/versioning/container_added/rest/con1/fcr%3Aversions/version_original.jsonld create mode 100644 src/test/resources/sample/versioning/container_added/rest/v_con1.jsonld create mode 100644 src/test/resources/sample/versioning/container_restored/rest/con1/child1.jsonld create mode 100644 src/test/resources/sample/versioning/container_restored/rest/con1/fcr%3Aversions.jsonld create mode 100644 src/test/resources/sample/versioning/container_restored/rest/con1/fcr%3Aversions/version_1.jsonld create mode 100644 src/test/resources/sample/versioning/container_restored/rest/con1/fcr%3Aversions/version_original.jsonld create mode 100644 src/test/resources/sample/versioning/container_restored/rest/con1/fcr%3Aversions/version_original/child1.jsonld create mode 100644 src/test/resources/sample/versioning/container_restored/rest/v_con1.jsonld create mode 100644 src/test/resources/sample/versioning/removed_in_head/fcrepo/rest/con1.jsonld create mode 100644 src/test/resources/sample/versioning/removed_in_head/fcrepo/rest/con1/fcr%3Aversions.jsonld create mode 100644 src/test/resources/sample/versioning/removed_in_head/fcrepo/rest/con1/fcr%3Aversions/version_original.jsonld create mode 100644 src/test/resources/sample/versioning/removed_in_head/fcrepo/rest/con1/fcr%3Aversions/version_original/con2.jsonld create mode 100644 src/test/resources/sample/versioning/unmodified/rest/v_con1.jsonld create mode 100644 src/test/resources/sample/versioning/unmodified/rest/v_con1/child1.jsonld create mode 100644 src/test/resources/sample/versioning/unmodified/rest/v_con1/fcr%3Aversions.jsonld create mode 100644 src/test/resources/sample/versioning/unmodified/rest/v_con1/fcr%3Aversions/version_original.jsonld create mode 100644 src/test/resources/sample/versioning/unmodified/rest/v_con1/fcr%3Aversions/version_original/child1.jsonld create mode 100644 src/test/resources/sample/versioning/unmodified/rest/v_con1/fcr%3Aversions/version_unchanged.jsonld create mode 100644 src/test/resources/sample/versioning/unmodified/rest/v_con1/fcr%3Aversions/version_unchanged/child1.jsonld diff --git a/src/main/java/org/fcrepo/importexport/common/BinaryImportException.java b/src/main/java/org/fcrepo/importexport/common/BinaryImportException.java new file mode 100644 index 00000000..7ce9ae76 --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/common/BinaryImportException.java @@ -0,0 +1,47 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.common; + +/** + * + * @author bbpennel + * + */ +public class BinaryImportException extends RuntimeException { + private static final long serialVersionUID = 1L; + + /** + * Constructor + * + * @param message message + */ + public BinaryImportException(final String message) { + super(message); + } + + /** + * Constructor + * + * @param message message + * @param cause cause + */ + public BinaryImportException(final String message, final Throwable cause) { + super(message, cause); + } + +} diff --git a/src/main/java/org/fcrepo/importexport/common/FcrepoConstants.java b/src/main/java/org/fcrepo/importexport/common/FcrepoConstants.java index 4c059b94..00a618b1 100644 --- a/src/main/java/org/fcrepo/importexport/common/FcrepoConstants.java +++ b/src/main/java/org/fcrepo/importexport/common/FcrepoConstants.java @@ -44,6 +44,7 @@ public abstract class FcrepoConstants { public static final String LDP_NAMESPACE = "http://www.w3.org/ns/ldp#"; public static final Resource CONTAINER = createResource(LDP_NAMESPACE + "Container"); public static final Property MEMBERSHIP_RESOURCE = createProperty(LDP_NAMESPACE + "membershipResource"); + public static final Property HAS_MEMBER_RELATION = createProperty(LDP_NAMESPACE + "hasMemberRelation"); public static final Property NON_RDF_SOURCE = createProperty(LDP_NAMESPACE + "NonRDFSource"); public static final Property RDF_SOURCE = createProperty(LDP_NAMESPACE + "RDFSource"); public static final Property CONTAINS = createProperty(LDP_NAMESPACE + "contains"); @@ -58,7 +59,9 @@ public abstract class FcrepoConstants { public static final String REPOSITORY_NAMESPACE = "http://fedora.info/definitions/v4/repository#"; public static final Resource INBOUND_REFERENCES = createResource(REPOSITORY_NAMESPACE + "InboundReferences"); public static final Resource PAIRTREE = createResource(REPOSITORY_NAMESPACE + "Pairtree"); + public static final Resource FEDORA_RESOURCE = createResource(REPOSITORY_NAMESPACE + "Resource"); public static final Resource REPOSITORY_ROOT = createResource(REPOSITORY_NAMESPACE + "RepositoryRoot"); + public static final Resource VERSION_RESOURCE = createResource(REPOSITORY_NAMESPACE + "Version"); public static final String BAG_INFO_FIELDNAME = "Bag-Info"; @@ -67,7 +70,10 @@ public abstract class FcrepoConstants { public static final Property LAST_MODIFIED_DATE = createProperty(REPOSITORY_NAMESPACE + "lastModified"); public static final Property LAST_MODIFIED_BY = createProperty(REPOSITORY_NAMESPACE + "lastModifiedBy"); + public static final String FCR_METADATA_PATH = "fcr:metadata"; + public static final String FCR_VERSIONS_PATH = "fcr:versions"; public static final Property HAS_VERSION = createProperty(REPOSITORY_NAMESPACE + "hasVersion"); + public static final Property HAS_VERSIONS = createProperty(REPOSITORY_NAMESPACE + "hasVersions"); public static final Property HAS_VERSION_LABEL = createProperty(REPOSITORY_NAMESPACE + "hasVersionLabel"); } diff --git a/src/main/java/org/fcrepo/importexport/common/ModelUtils.java b/src/main/java/org/fcrepo/importexport/common/ModelUtils.java new file mode 100644 index 00000000..31d09f68 --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/common/ModelUtils.java @@ -0,0 +1,79 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.common; + +import static org.apache.jena.riot.RDFLanguages.contentTypeToLang; + +import java.io.IOException; +import java.io.InputStream; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFDataMgr; +import org.fcrepo.importexport.importer.SubjectMappingStreamRDF; +import org.fcrepo.importexport.importer.VersionSubjectMappingStreamRDF; + +/** + * Utilities for manipulating rdf models + * + * @author bbpennel + * + */ +public abstract class ModelUtils { + + /** + * Parses an input stream of RDF in the configured RDF language into a model, remapping subjects to the configured + * destination URL. This includes removal of version path components if configured. + * + * @param in RDF input stream + * @param config config + * @return parsed model + * @throws IOException Thrown if the stream cannot be parsed + */ + public static Model mapRdfStreamToNonversionedSubjects(final InputStream in, final Config config) + throws IOException { + final SubjectMappingStreamRDF mapper; + if (config.includeVersions()) { + mapper = new VersionSubjectMappingStreamRDF(config.getSource(), config.getDestination()); + } else { + mapper = new SubjectMappingStreamRDF(config.getSource(), config.getDestination()); + } + + try (final InputStream in2 = in) { + RDFDataMgr.parse(mapper, in2, contentTypeToLang(config.getRdfLanguage())); + } + return mapper.getModel(); + } + + /** + * Parses an input stream of RDF in the configured RDF language into a model, remapping subjects to the configured + * destination URL. + * + * @param in RDF input stream + * @param config config + * @return parsed model + * @throws IOException Thrown if the stream cannot be parsed + */ + public static Model mapRdfStream(final InputStream in, final Config config) throws IOException { + final SubjectMappingStreamRDF mapper = new SubjectMappingStreamRDF(config.getSource(), + config.getDestination()); + try (final InputStream in2 = in) { + RDFDataMgr.parse(mapper, in2, contentTypeToLang(config.getRdfLanguage())); + } + return mapper.getModel(); + } +} diff --git a/src/main/java/org/fcrepo/importexport/common/ResourceGoneRuntimeException.java b/src/main/java/org/fcrepo/importexport/common/ResourceGoneRuntimeException.java new file mode 100644 index 00000000..e867840a --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/common/ResourceGoneRuntimeException.java @@ -0,0 +1,58 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.common; + +import java.net.URI; + +import org.fcrepo.client.FcrepoResponse; + +/** + * Exception thrown when a resource is gone, and checks to see if the resource has a tombstone + * + * @author bbpennel + */ +public class ResourceGoneRuntimeException extends RuntimeException { + private static final long serialVersionUID = 1L; + + final URI resourceUri; + final URI tombstone; + + /** + * Constructor + * + * @param response Response which triggered the exception + */ + public ResourceGoneRuntimeException(final FcrepoResponse response) { + tombstone = response.getLinkHeaders("hasTombstone").get(0); + resourceUri = response.getUrl(); + } + + /** + * @return the resourceUri + */ + public URI getResourceUri() { + return resourceUri; + } + + /** + * @return the tombstone + */ + public URI getTombstone() { + return tombstone; + } +} diff --git a/src/main/java/org/fcrepo/importexport/common/TransferProcess.java b/src/main/java/org/fcrepo/importexport/common/TransferProcess.java index c6a29043..f2fa9257 100644 --- a/src/main/java/org/fcrepo/importexport/common/TransferProcess.java +++ b/src/main/java/org/fcrepo/importexport/common/TransferProcess.java @@ -168,7 +168,7 @@ public static File directoryForContainer(final URI uri, final String sourcePath, * @return the map */ public static Map getSha1FileMap(final File baseDir, final Path manifestFile) { - final Map sha1FileMap = new HashMap(); + final Map sha1FileMap = new HashMap<>(); try (final Stream stream = Files.lines(manifestFile)) { stream.forEach(l -> { final String[] manifestTokens = l.split(BAGIT_CHECKSUM_DELIMITER); diff --git a/src/main/java/org/fcrepo/importexport/common/URITranslationUtil.java b/src/main/java/org/fcrepo/importexport/common/URITranslationUtil.java new file mode 100644 index 00000000..6e4de8fc --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/common/URITranslationUtil.java @@ -0,0 +1,120 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.common; + +import static org.fcrepo.importexport.common.FcrepoConstants.FCR_VERSIONS_PATH; + +import java.io.File; +import java.net.URI; + +/** + * Utility for creating or manipulating uris + * + * @author bbpennel + * + */ +public abstract class URITranslationUtil { + + /** + * Builds the repository URI for the given file + * + * @param f file to build URI for + * @param config config + * @return URI for file + */ + public static URI uriForFile(final File f, final Config config) { + // get path of file relative to the data directory + String relative = config.getBaseDirectory().toPath().relativize(f.toPath()).toString(); + relative = TransferProcess.decodePath(relative); + + // rebase the path on the destination uri (translating source/destination if needed) + if ( config.getSource() != null && config.getDestination() != null ) { + relative = baseURI(config.getSource()) + relative; + relative = relative.replaceFirst(config.getSource().toString(), config.getDestination().toString()); + } else { + relative = baseURI(config.getResource()) + relative; + } + + // for exported RDF, just remove the ".extension" and you have the encoded path + if (relative.endsWith(config.getRdfExtension())) { + relative = relative.substring(0, relative.length() - config.getRdfExtension().length()); + } + return URI.create(relative); + } + + /** + * Adds relative path to uri + * + * @param uri base uri + * @param path relative path to add + * @return joined uri + */ + public static URI addRelativePath(final URI uri, final String path) { + final String base = uri.toString(); + + if (base.charAt(base.length() - 1) == '/') { + if (path.charAt(0) == '/') { + return URI.create(base + path.substring(1, path.length())); + } + return URI.create(base + path); + } else if (path.charAt(0) == '/') { + return URI.create(base + path); + } + + return URI.create(base + "/" + path); + } + + private static String baseURI(final URI uri) { + final String base = uri.toString().replaceFirst(uri.getPath() + "$", ""); + return (base.endsWith("/")) ? base : base + "/"; + } + + /** + * Remaps the given uri to its expected uri within the destination repository + * + * @param uri resource uri + * @param sourceURI source uri + * @param destinationURI destination base uri + * @return remapped resource uri + */ + public static URI remapResourceUri(final URI uri, final URI sourceURI, final URI destinationURI) { + return URI.create(remapResourceUri(uri.toString(), sourceURI == null ? null : sourceURI.toString(), + destinationURI == null ? null : destinationURI.toString())); + } + + /** + * Remaps the given uri to its expected uri within the destination repository + * + * @param uri resource uri + * @param sourceURI source uri + * @param destinationURI destination base uri + * @return remapped resource uri + */ + public static String remapResourceUri(final String uri, final String sourceURI, final String destinationURI) { + String remapped = uri; + if (remapped.contains(FCR_VERSIONS_PATH)) { + remapped = remapped.replaceFirst("/fcr:versions/[^/]+", ""); + } + if (sourceURI != null && destinationURI != null + && uri.startsWith(sourceURI)) { + remapped = remapped.replaceFirst(sourceURI, destinationURI); + } + + return remapped; + } +} diff --git a/src/main/java/org/fcrepo/importexport/importer/ChronologicalImportEventIterator.java b/src/main/java/org/fcrepo/importexport/importer/ChronologicalImportEventIterator.java new file mode 100644 index 00000000..dda68f57 --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/importer/ChronologicalImportEventIterator.java @@ -0,0 +1,304 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static java.nio.file.FileVisitResult.CONTINUE; +import static org.fcrepo.importexport.common.FcrepoConstants.CREATED_DATE; +import static org.fcrepo.importexport.common.FcrepoConstants.LAST_MODIFIED_DATE; +import static org.fcrepo.importexport.common.FcrepoConstants.NON_RDF_SOURCE; +import static org.fcrepo.importexport.common.FcrepoConstants.RDF_TYPE; +import static org.fcrepo.importexport.common.ModelUtils.mapRdfStream; +import static org.slf4j.LoggerFactory.getLogger; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import org.apache.jena.datatypes.xsd.XSDDateTime; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ResIterator; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.Statement; +import org.fcrepo.importexport.common.Config; +import org.fcrepo.importexport.common.URITranslationUtil; +import org.slf4j.Logger; + +/** + * Iterates through a deduplicated set of import events in chronological order based on timestamp + * + * Generates the list of events from the configured import directory. Events are either resource import or version + * creation events. They are sorted chronologically by timestamp, using the created timestamp for the original + * version of the resource, or the last modified timestamp for subsequent versions. Unmodified versions of a + * resource are also deduplicated if another version with the same last modified timestamp is present. + * + * @author bbpennel + * + */ +public class ChronologicalImportEventIterator implements Iterator { + + private static final Logger logger = getLogger(ChronologicalImportEventIterator.class); + + private final Config config; + private Queue eventQueue; + final File importBaseDirectory; + + /** + * Constructs a new ChronologicalImportEventIterator from the given base directory + * + * @param importBaseDirectory base directory where resources are extracted from + * @param config config + */ + public ChronologicalImportEventIterator(final File importBaseDirectory, final Config config) { + this.config = config; + this.importBaseDirectory = importBaseDirectory; + } + + @Override + public boolean hasNext() { + if (eventQueue == null) { + try { + eventQueue = generateEventQueue(); + } catch (final IOException e) { + throw new RuntimeException("Failed to read resources for import from specified directory " + + importBaseDirectory, e); + } + } + + return eventQueue.size() > 0; + } + + @Override + public ImportEvent next() { + return eventQueue.remove(); + } + + /** + * Generates the list of events from the configured import directory. Events are either resource import or version + * creation events. They are sorted chronologically by timestamp, using the created timestamp for the original + * version of the resource, or the last modified timestamp for subsequent versions. Unmodified versions of a + * resource are also deduplicated if another version with the same last modified timestamp is present. + * + * @throws IOException thrown if the directory cannot be walked + */ + private Queue generateEventQueue() throws IOException { + final ChronologicalUriExtractingFileVisitor treeWalker = + new ChronologicalUriExtractingFileVisitor(this.config); + Files.walkFileTree(importBaseDirectory.toPath(), treeWalker); + + // Retrieve list of events from the configured directory + final List eventList = treeWalker.getEvents(); + + // If including versions, then remove unmodified duplicates and setup sort keys + if (config.includeVersions()) { + final Map> resourcesGroupedByUri = treeWalker.getResourcesGroupedByUri(); + prepareResources(eventList, resourcesGroupedByUri); + } + + // Sort the events + sortEventsChronologically(eventList); + + final Queue events = new ArrayDeque<>(); + events.addAll(eventList); + + return events; + } + + private void prepareResources(final List eventList, + final Map> resourcesGroupedByUri) { + + resourcesGroupedByUri.entrySet().forEach(uriGroup -> { + final List rescGroup = uriGroup.getValue(); + if (rescGroup.size() <= 1) { + return; + } + + // Sort the versions of this resource by last modified + rescGroup.sort(new Comparator() { + @Override + public int compare(final ImportEvent o1, final ImportEvent o2) { + return new Long(o1.getLastModified()).compareTo(o2.getLastModified()); + } + }); + + // Remove unmodified resources and switch sort key to last modified for all versions after original + for (int i = 1; i < rescGroup.size(); i++) { + final ImportResource resc = rescGroup.get(i); + final long previousLastModified = rescGroup.get(i - 1).getLastModified(); + + // Remove the unmodified resource from the total event list + if (resc.getLastModified() == previousLastModified) { + eventList.remove(resc); + } else { + resc.setTimestamp(resc.getLastModified()); + } + } + }); + } + + /** + * Sorts the events list chronologically by timestamp + * + * @param eventList + */ + private List sortEventsChronologically(final List eventList) { + eventList.sort(new Comparator() { + @Override + public int compare(final ImportEvent o1, final ImportEvent o2) { + return new Long(o1.getTimestamp()).compareTo(o2.getTimestamp()); + } + }); + + return eventList; + } + + private class ChronologicalUriExtractingFileVisitor extends SimpleFileVisitor { + + private List resources; + private Map> resourcesGroupedByUri; + + private final Config config; + + public ChronologicalUriExtractingFileVisitor(final Config config) { + resources = new ArrayList<>(); + resourcesGroupedByUri = new HashMap<>(); + this.config = config; + } + + public List getEvents() { + return resources; + } + + public Map> getResourcesGroupedByUri() { + return resourcesGroupedByUri; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + // Skip over files other than descriptions + if (!file.toString().endsWith(config.getRdfExtension())) { + return CONTINUE; + } + + // Create events for versions of this resource + if (config.includeVersions() + && file.toString().endsWith("fcr%3Aversions" + config.getRdfExtension())) { + addVersionsEvents(file.toFile()); + return CONTINUE; + } + + // If versioning is excluded then skip all past versions of resources + if (!config.includeVersions() && file.toString().contains("fcr%3Aversions")) { + return CONTINUE; + } + + final File rdfFile = file.toFile(); + final Model model = mapRdfStream(new FileInputStream(rdfFile), config); + + // Determine the URI for this resource depending on if it is a binary or not + final URI resourceUri; + final ResIterator binaryResources = model.listResourcesWithProperty(RDF_TYPE, NON_RDF_SOURCE); + try { + if (binaryResources.hasNext()) { + resourceUri = URI.create(binaryResources.next().getURI()); + } else { + resourceUri = URITranslationUtil.uriForFile(rdfFile, config); + } + } finally { + binaryResources.close(); + } + + // Store the resource along with its last modified date for sorting later + final Resource resc = model.getResource(resourceUri.toString()); + final Statement lastModStmt = resc.getProperty(LAST_MODIFIED_DATE); + final Statement createdStmt = resc.getProperty(CREATED_DATE); + + final long lastModified = lastModStmt == null ? 0L : getTimestampFromProperty(lastModStmt); + final long created = createdStmt == null ? 0L : getTimestampFromProperty(createdStmt); + + final ImportResource impResc = new ImportResource(resourceUri, rdfFile, created, lastModified, config); + + resources.add(impResc); + + logger.debug("Added resource for import {}", impResc.getUri()); + + // If including versions, then store a map of destination uris to all resources with the same uris + if (config.includeVersions()) { + final String groupUri = impResc.getMappedUri().toString(); + List rescsForUri = resourcesGroupedByUri.get(groupUri); + if (rescsForUri == null) { + rescsForUri = new ArrayList<>(); + resourcesGroupedByUri.put(groupUri, rescsForUri); + } + rescsForUri.add(impResc); + } + + return CONTINUE; + } + + private void addVersionsEvents(final File versionsFile) throws IOException { + final Model model = mapRdfStream(new FileInputStream(versionsFile), config); + + final VersionDiffDeletionGenerator deletionGenerator = new VersionDiffDeletionGenerator(config); + resources.addAll(deletionGenerator.generateImportDeletions(model)); + + // Add versions in after deletions + resources.addAll(getVersionEvents(model)); + } + + private List getVersionEvents(final Model model) { + final List versions = new ArrayList<>(); + + final ResIterator vRescIt = model.listResourcesWithProperty(CREATED_DATE); + try { + while (vRescIt.hasNext()) { + final Resource vResc = vRescIt.next(); + final URI rescUri = URI.create(vResc.getURI()); + final long time = getTimestampFromProperty(vResc.getProperty(CREATED_DATE)); + + final ImportVersion impVersion = new ImportVersion(rescUri, time, config); + versions.add(impVersion); + + logger.debug("Added version {}", impVersion.getUri()); + } + } finally { + vRescIt.close(); + } + + return versions; + } + + private long getTimestampFromProperty(final Statement stmt) { + final XSDDateTime dateTime = (XSDDateTime) stmt.getLiteral().getValue(); + return dateTime.asCalendar().getTimeInMillis(); + } + } +} diff --git a/src/main/java/org/fcrepo/importexport/importer/ImportDeletion.java b/src/main/java/org/fcrepo/importexport/importer/ImportDeletion.java new file mode 100644 index 00000000..01b98ca8 --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/importer/ImportDeletion.java @@ -0,0 +1,43 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import java.net.URI; + +import org.fcrepo.importexport.common.Config; + +/** + * An event for the deletion of a resource that existed in a previous version of a tree + * + * @author bbpennel + * + */ +public class ImportDeletion extends ImportEvent { + + /** + * Construct ImportDeletion + * + * @param uri uri of the resource being deleted + * @param timestamp timestamp for the deletion + * @param config config + */ + public ImportDeletion(final URI uri, final long timestamp, final Config config) { + super(uri, timestamp, timestamp, config); + } + +} diff --git a/src/main/java/org/fcrepo/importexport/importer/ImportEvent.java b/src/main/java/org/fcrepo/importexport/importer/ImportEvent.java new file mode 100644 index 00000000..760add6d --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/importer/ImportEvent.java @@ -0,0 +1,111 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static org.fcrepo.importexport.common.URITranslationUtil.remapResourceUri; + +import java.net.URI; + +import org.fcrepo.importexport.common.Config; + +/** + * An object representing an event to be executed when importing a resource + * + * @author bbpennel + * + */ +public abstract class ImportEvent { + + protected final URI uri; + protected final URI mappedUri; + protected final Config config; + protected final long lastModified; + protected final long created; + protected long timestamp; + + /** + * Constructs an ImportEvent + * + * @param uri uri of the resource affected by the event + * @param created created timestamp + * @param lastModified last modified timestamp + * @param config config + */ + public ImportEvent(final URI uri, final long created, final long lastModified, + final Config config) { + this.config = config; + this.uri = uri; + this.created = created; + this.lastModified = lastModified; + this.timestamp = created; + this.mappedUri = remapResourceUri(uri, config.getSource(), config.getDestination()); + } + + /** + * Get the original URI for this resource + * + * @return the uri + */ + public URI getUri() { + return uri; + } + + /** + * Get the URI for this resource remapped for the destination repository + * + * @return the mapped uri + */ + public URI getMappedUri() { + return mappedUri; + } + + + /** + * @return the lastModified timestamp + */ + public long getLastModified() { + return lastModified; + } + + + /** + * @return the created timestamp + */ + public long getCreated() { + return created; + } + + /** + * @return the comparable timestamp for this resource + */ + public long getTimestamp() { + return timestamp; + } + + /** + * @param timestamp the timestamp to set + */ + public void setTimestamp(final long timestamp) { + this.timestamp = timestamp; + } + + @Override + public String toString() { + return uri.toString(); + } +} diff --git a/src/main/java/org/fcrepo/importexport/importer/ImportResource.java b/src/main/java/org/fcrepo/importexport/importer/ImportResource.java new file mode 100644 index 00000000..e5273efc --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/importer/ImportResource.java @@ -0,0 +1,153 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static org.fcrepo.importexport.common.FcrepoConstants.BINARY_EXTENSION; +import static org.fcrepo.importexport.common.FcrepoConstants.EXTERNAL_RESOURCE_EXTENSION; +import static org.fcrepo.importexport.common.FcrepoConstants.FCR_METADATA_PATH; +import static org.fcrepo.importexport.common.FcrepoConstants.RDF_TYPE; +import static org.fcrepo.importexport.common.ModelUtils.mapRdfStreamToNonversionedSubjects; +import static org.fcrepo.importexport.common.URITranslationUtil.addRelativePath; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.Resource; +import org.fcrepo.importexport.common.Config; +import org.fcrepo.importexport.common.TransferProcess; + + +/** + * An event for importing a resource + * + * @author bbpennel + * + */ +public class ImportResource extends ImportEvent { + + private URI descriptionUri; + private Model model; + private Resource resource; + private File binary; + private File descriptionFile; + + /** + * Construct new ImportResource + * + * @param uri uri of resource + * @param descriptionFile description file for resource + * @param created created timestamp + * @param lastModified last modified timestamp for resource + * @param config config + */ + public ImportResource(final URI uri, final File descriptionFile, final long created, + final long lastModified, final Config config) { + super(uri, created, lastModified, config); + this.descriptionFile = descriptionFile; + } + + /** + * Get the URI for metadata for this resource + * + * @return uri of the RDF endpoint for resource + */ + public URI getDescriptionUri() { + if (descriptionUri == null) { + if (isBinary()) { + descriptionUri = addRelativePath(getMappedUri(), FCR_METADATA_PATH); + } else { + descriptionUri = getMappedUri(); + } + } + return descriptionUri; + } + + /** + * Test if this resource is a binary + * + * @return true if resource is a binary + */ + public boolean isBinary() { + return getBinary().exists(); + } + + /** + * Get the binary file for this resource + * + * @return the binary for this resource or null if not found + */ + public File getBinary() { + if (binary == null) { + binary = TransferProcess.fileForURI(uri, config.getSourcePath(), + config.getDestinationPath(), config.getBaseDirectory(), BINARY_EXTENSION); + if (!binary.exists()) { + binary = TransferProcess.fileForURI(uri, config.getSourcePath(), + config.getDestinationPath(), config.getBaseDirectory(), EXTERNAL_RESOURCE_EXTENSION); + } + } + return binary; + } + + /** + * Get the file containing metadata for this resource + * + * @return returns the file containing this resource's description + */ + public File getDescriptionFile() { + return descriptionFile; + } + + /** + * Get the model containing properties assigned to this resource + * + * @return model containing properties assigned to this resource + */ + public Model getModel() { + if (model == null) { + final File mdFile = getDescriptionFile(); + if (mdFile == null) { + return null; + } + try { + model = mapRdfStreamToNonversionedSubjects(new FileInputStream(mdFile), config); + } catch (IOException e) { + throw new RuntimeException("Failed to read model for " + uri, e); + } + } + return model; + } + + /** + * Get the resource representing this ImportResource from its model + * + * @return rdf resource for this resource + */ + public Resource getResource() { + if (resource == null) { + final Model model = getModel(); + if (model == null) { + return null; + } + resource = model.listResourcesWithProperty(RDF_TYPE).next(); + } + return resource; + } +} diff --git a/src/main/java/org/fcrepo/importexport/importer/ImportVersion.java b/src/main/java/org/fcrepo/importexport/importer/ImportVersion.java new file mode 100644 index 00000000..222e0c8f --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/importer/ImportVersion.java @@ -0,0 +1,69 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import java.net.URI; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.fcrepo.importexport.common.Config; + +/** + * An event for importing a version of a resource + * + * @author bbpennel + * + */ +public class ImportVersion extends ImportEvent { + final private static Pattern versionUriPattern = Pattern.compile(".+/fcr:versions/([^/]+)(/.+)?"); + + private final String label; + + /** + * Constructs a ImportVersion object + * + * @param uri uri of the resource being versioned + * @param created created timestamp of the version + * @param config config + */ + public ImportVersion(final URI uri, final long created, final Config config) { + super(uri, created, created, config); + + final Matcher versionUriMatcher = versionUriPattern.matcher(uri.toString()); + + if (versionUriMatcher.matches()) { + this.label = versionUriMatcher.group(1); + } else { + throw new RuntimeException("Version for resource " + uri + " does not provide a required label"); + } + } + + /** + * The label of the version bring imported + * + * @return label + */ + public String getLabel() { + return label; + } + + @Override + public long getTimestamp() { + return created; + } +} diff --git a/src/main/java/org/fcrepo/importexport/importer/Importer.java b/src/main/java/org/fcrepo/importexport/importer/Importer.java index 32c8e81f..36998b66 100644 --- a/src/main/java/org/fcrepo/importexport/importer/Importer.java +++ b/src/main/java/org/fcrepo/importexport/importer/Importer.java @@ -17,19 +17,16 @@ */ package org.fcrepo.importexport.importer; +import static java.nio.file.FileVisitResult.CONTINUE; import static java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME; -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toList; -import static org.apache.jena.rdf.model.ResourceFactory.createProperty; import static org.apache.jena.rdf.model.ResourceFactory.createResource; -import static org.apache.jena.riot.RDFLanguages.contentTypeToLang; -import static org.fcrepo.importexport.common.FcrepoConstants.BINARY_EXTENSION; -import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINER; +import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; import static org.fcrepo.importexport.common.FcrepoConstants.CREATED_BY; import static org.fcrepo.importexport.common.FcrepoConstants.CREATED_DATE; import static org.fcrepo.importexport.common.FcrepoConstants.DESCRIBEDBY; -import static org.fcrepo.importexport.common.FcrepoConstants.EXTERNAL_RESOURCE_EXTENSION; +import static org.fcrepo.importexport.common.FcrepoConstants.FCR_VERSIONS_PATH; +import static org.fcrepo.importexport.common.FcrepoConstants.HAS_MEMBER_RELATION; import static org.fcrepo.importexport.common.FcrepoConstants.HAS_MESSAGE_DIGEST; import static org.fcrepo.importexport.common.FcrepoConstants.HAS_MIME_TYPE; import static org.fcrepo.importexport.common.FcrepoConstants.HAS_SIZE; @@ -41,10 +38,13 @@ import static org.fcrepo.importexport.common.FcrepoConstants.RDF_SOURCE; import static org.fcrepo.importexport.common.FcrepoConstants.RDF_TYPE; import static org.fcrepo.importexport.common.FcrepoConstants.REPOSITORY_NAMESPACE; +import static org.fcrepo.importexport.common.FcrepoConstants.REPOSITORY_ROOT; +import static org.fcrepo.importexport.common.ModelUtils.mapRdfStream; import static org.fcrepo.importexport.common.TransferProcess.fileForBinary; import static org.fcrepo.importexport.common.TransferProcess.fileForExternalResources; import static org.fcrepo.importexport.common.TransferProcess.fileForURI; import static org.fcrepo.importexport.common.TransferProcess.isRepositoryRoot; +import static org.fcrepo.importexport.common.URITranslationUtil.addRelativePath; import static org.fcrepo.importexport.common.UriUtils.withSlash; import static org.slf4j.LoggerFactory.getLogger; @@ -55,38 +55,41 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.net.URISyntaxException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.ZonedDateTime; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.time.ZoneId; +import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.io.IOUtils; +import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Property; +import org.apache.jena.rdf.model.RDFNode; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.rdf.model.StmtIterator; import org.fcrepo.client.FcrepoClient; +import org.fcrepo.client.FcrepoClient.FcrepoClientBuilder; import org.fcrepo.client.FcrepoOperationFailedException; import org.fcrepo.client.FcrepoResponse; import org.fcrepo.client.PutBuilder; import org.fcrepo.importexport.common.AuthenticationRequiredRuntimeException; +import org.fcrepo.importexport.common.BinaryImportException; import org.fcrepo.importexport.common.Config; +import org.fcrepo.importexport.common.ResourceGoneRuntimeException; import org.fcrepo.importexport.common.ResourceNotFoundRuntimeException; import org.fcrepo.importexport.common.TransferProcess; - -import org.apache.commons.io.IOUtils; -import org.apache.jena.rdf.model.Model; -import org.apache.jena.rdf.model.NodeIterator; -import org.apache.jena.rdf.model.RDFNode; -import org.apache.jena.rdf.model.ResIterator; -import org.apache.jena.rdf.model.Resource; -import org.apache.jena.rdf.model.Statement; -import org.apache.jena.rdf.model.StmtIterator; -import org.apache.jena.riot.RDFDataMgr; -import org.apache.jena.riot.RiotException; import org.slf4j.Logger; import gov.loc.repository.bagit.domain.Bag; @@ -99,61 +102,48 @@ * @author lsitu * @author awoods * @author escowles + * @author bbpennel * @since 2016-08-29 */ public class Importer implements TransferProcess { + private static final Logger logger = getLogger(Importer.class); private Config config; - protected FcrepoClient.FcrepoClientBuilder clientBuilder; - private final List membershipResources = new ArrayList<>(); - private final List relatedResources = new ArrayList<>(); - private final List importedResources = new ArrayList<>(); - private URI repositoryRoot = null; + protected FcrepoClientBuilder clientBuilder; - private Bag bag; + private Logger importLogger; - private MessageDigest sha1; + final Map versionedLabels; - private Map sha1FileMap; + private final Map> uriToMembershipRelations; - private Logger importLogger; - private AtomicLong successCount = new AtomicLong(); // set to zero at start + private final Map sha1FileMap; - /** - * A directory within the metadata directory that serves as the - * root of the resource being imported. If the export directory - * contains /fcrepo/rest/one/two/three and we're importing - * the resource at /fcrepo/rest/one/two, this stores that path. - */ - protected File importContainerDirectory; + private URI repositoryRoot = null; + private AtomicLong successCount = new AtomicLong(); // set to zero at start /** - * Constructor that takes the Import/Export configuration + * Construct an importer * - * @param config for import - * @param clientBuilder for sending resources to Fedora + * @param config config + * @param clientBuilder fcrepo client builder */ - public Importer(final Config config, final FcrepoClient.FcrepoClientBuilder clientBuilder) { + public Importer(final Config config, final FcrepoClientBuilder clientBuilder) { + this.config = config; this.clientBuilder = clientBuilder; this.importLogger = config.getAuditLog(); + this.versionedLabels = new HashMap<>(); + this.uriToMembershipRelations = new HashMap<>(); + if (config.getBagProfile() == null) { - this.bag = null; - this.sha1 = null; this.sha1FileMap = null; } else { - - try { - final File bagdir = config.getBaseDirectory().getParentFile(); - // TODO: Maybe use this once we get an updated release of bagit-java library - //if (verifyBag(bagdir)) { - final Path manifestPath = Paths.get(bagdir.getAbsolutePath()).resolve("manifest-sha1.txt"); - this.sha1FileMap = TransferProcess.getSha1FileMap(bagdir, manifestPath); - this.sha1 = MessageDigest.getInstance("SHA-1"); - // } - } catch (NoSuchAlgorithmException e) { - // never happens with known algorithm names - } + final File bagdir = config.getBaseDirectory().getParentFile(); + // TODO: Maybe use this once we get an updated release of bagit-java library + //if (verifyBag(bagdir)) { + final Path manifestPath = Paths.get(bagdir.getAbsolutePath()).resolve("manifest-sha1.txt"); + this.sha1FileMap = TransferProcess.getSha1FileMap(bagdir, manifestPath); } } @@ -180,274 +170,195 @@ public void run() { } private void processImport(final URI resource) { - importedResources.add(resource); - final File importContainerMetadataFile = fileForContainerURI(resource); - importContainerDirectory = directoryForContainer(resource); - - // clean up the membership resources that were imported. - membershipResources.clear(); - discoverMembershipResources(importContainerDirectory); + final File importContainerDirectory = importContainerMetadataFile.getParentFile(); - // In order to get the dates right, we need to create resources, *then* set their RDF since - // creation of children screws up the dates of parents. try { - ensureExists(resource); + // Generate list of resources + discoverMembershipResources(importContainerDirectory); + + final Iterator rescIt = new ChronologicalImportEventIterator( + importContainerDirectory, config); + while (rescIt.hasNext()) { + final ImportEvent impEvent = rescIt.next(); + + if (impEvent instanceof ImportResource) { + importResource((ImportResource) impEvent); + } else if (impEvent instanceof ImportVersion) { + final ImportVersion version = (ImportVersion) impEvent; + createVersion(version.getMappedUri(), version.getLabel()); + } else if (impEvent instanceof ImportDeletion) { + final ImportDeletion deletion = (ImportDeletion) impEvent; + deleteRemovedVersion(deletion.getMappedUri()); + } + } } catch (FcrepoOperationFailedException | IOException e) { - throw new RuntimeException("Unable to create placeholder " + resource, e); + throw new RuntimeException("Failed to import", e); } + } - importDirectory(importContainerDirectory); - - logger.debug("Importing resource {} for file {}", resource, importContainerMetadataFile.getPath()); - - // discover the related resources in the resource being imported. - if (!importContainerMetadataFile.exists()) { - logger.debug("No container exists in the metadata directory {} for the requested resource {}," - + " importing all contained resources instead.", importContainerMetadataFile.getPath(), - config.getResource()); + private void importResource(final ImportResource resc) throws FcrepoOperationFailedException, IOException { + if (resc.isBinary()) { + importBinaryResource(resc); } else { - logger.debug("Parsing membership resource in file {}", importContainerMetadataFile.getAbsolutePath()); - parseMembershipResources(importContainerMetadataFile); - importMembershipResources(); - importRelatedResources(); - importFile(importContainerMetadataFile); + importContainerResource(resc); } - } - private void discoverMembershipResources(final File dir) { - if (dir.listFiles() != null) { - stream(dir.listFiles()).filter(File::isFile).forEach(f -> parseMembershipResources(f)); - stream(dir.listFiles()).filter(File::isDirectory).forEach(d -> discoverMembershipResources(d)); + private void importContainerResource(final ImportResource resc) throws FcrepoOperationFailedException, + IOException { + if (!isSkippableContainer(resc)) { + try { + importDescription(resc); + } catch (final ResourceGoneRuntimeException e) { + if (config.overwriteTombstones()) { + deleteTombstone(e.getResourceUri(), e.getTombstone()); + importDescription(resc); + } else { + throw e; + } + } } } - private void parseMembershipResources(final File f) { - // skip files that aren't RDF - if (!f.getName().endsWith(config.getRdfExtension())) { + private boolean isSkippableContainer(final ImportResource impResource) { + final Resource resource = impResource.getResource(); + return resource == null + || resource.hasProperty(RDF_TYPE, REPOSITORY_ROOT) + || resource.hasProperty(RDF_TYPE, PAIRTREE); + } + + private void importBinaryResource(final ImportResource resc) throws FcrepoOperationFailedException, IOException { + if (!config.isIncludeBinaries()) { return; } try { - final Model model = parseStream(new FileInputStream(f)); - if (model.contains(null, MEMBERSHIP_RESOURCE, (RDFNode)null)) { - model.listObjectsOfProperty(MEMBERSHIP_RESOURCE).forEachRemaining(node -> { - logger.info("Membership resource: {}", node); - membershipResources.add(URI.create(node.toString())); - }); - } - - // Discover all the related resources with member predicates. Those related resources that aren't imported - // during importing the targeted resource are in other container hierarchy that need to handle specifically. - // The related resources referenced by default predicate could be ignored. - for (final String p : config.getPredicates()) { - if (!p.equals(CONTAINS.toString())) { - for (final NodeIterator it = model.listObjectsOfProperty(createProperty(p)); it.hasNext();) { - final String uri = it.nextNode().toString(); - - logger.debug("Discovered related resource {} for source {}.", uri, config.getSource()); - if (!importedResources.contains(URI.create(uri))) { - // add related resource to list, exclude those that are already imported - final URI resURI = URI.create(uri); - if ((fileForContainerURI(resURI).exists())) { - relatedResources.add(URI.create(uri)); - - logger.debug("Added related resource {}", uri); - } else if ((fileForBinaryURI(resURI, false).exists() || fileForBinaryURI(resURI, true) - .exists())) { - importedResources.add(URI.create(uri)); - - // The binary file will be imported when the non-RDF metadata file is being imported. - importDirectory(directoryForContainer(resURI)); - } - } - } - } - } - } catch (final IOException e) { - throw new RuntimeException("Error reading file: " + f.getAbsolutePath() + ": " + e.toString()); - } catch (final RiotException e) { - throw new RuntimeException("Error parsing RDF: " + f.getAbsolutePath() + ": " + e.toString()); + importBinaryFile(resc); + // update metadata + importDescription(resc); + } catch (final ResourceGoneRuntimeException e) { + deleteTombstone(e.getResourceUri(), e.getTombstone()); + + importBinaryFile(resc); + importDescription(resc); + } catch (final BinaryImportException e) { + logger.error(e.getMessage()); } } - private void importRelatedResources() { - if (relatedResources.size() > 0) { - final List referenceResources = relatedResources.stream().collect(toList()); - relatedResources.clear(); - // loop through for nested related resources - referenceResources.stream().forEach(uri -> { - logger.info("Importing related resources {} ...", uri); - processImport(uri); - }); + private void createVersion(final URI uri, final String label) { + final URI versionsUri = addRelativePath(uri, FCR_VERSIONS_PATH); + try { + final FcrepoResponse response = client().post(versionsUri) + .slug(label) + .perform(); + + if (response.getStatusCode() == 201) { + logger.info("Created version {} of {}", label, uri); + importLogger.info("Created version {} of {}", label, uri); + successCount.incrementAndGet(); + } else { + throw new RuntimeException("Error creating version " + label + " of " + uri + + " (" + response.getStatusCode() + "): " + IOUtils.toString(response.getBody())); + } + } catch (FcrepoOperationFailedException | IOException e) { + throw new RuntimeException("Error creating version " + label + " of " + uri, e); } } - private void importMembershipResources() { - membershipResources.stream().forEach(uri -> importMembershipResource(uri)); + private void deleteRemovedVersion(final URI uri) { + try (final FcrepoResponse response = client().delete(uri).perform()) { + if (response.getStatusCode() == 204) { + deleteTombstone(uri, null); + logger.info("Delete removed version {}", uri); + importLogger.info("Delete removed version {}", uri); + successCount.incrementAndGet(); + } else { + final String bodyString = response.getBody() == null ? "" : + IOUtils.toString(response.getBody(), "UTF-8"); + throw new RuntimeException("Error deleting removed " + uri + + " (" + response.getStatusCode() + "): " + bodyString); + } + } catch (FcrepoOperationFailedException | IOException e) { + throw new RuntimeException("Error deleting of " + uri, e); + } } - private void importMembershipResource(final URI uri) { - final File f = fileForContainerURI(uri); + private void importDescription(final ImportResource resc) { + final URI destinationUri = resc.getMappedUri(); + final String descriptionPath = resc.getDescriptionFile().getAbsolutePath(); try { - final Model diskModel = parseStream(new FileInputStream(f)); - final Model repoModel = parseStream(client().get(uri).perform().getBody()); - final FcrepoResponse response = importContainer(uri, sanitize(diskModel.difference(repoModel))); + final FcrepoResponse response = descriptionBuilder(resc).preferLenient().perform(); + if (response.getStatusCode() == 401) { - importLogger.error("Error importing {} to {}, 401 Unauthorized", f.getAbsolutePath(), uri); + importLogger.error("Error importing {} to {}, 401 Unauthorized", + descriptionPath, destinationUri); throw new AuthenticationRequiredRuntimeException(); + } else if (response.getStatusCode() == 410 && config.overwriteTombstones()) { + throw new ResourceGoneRuntimeException(response); } else if (response.getStatusCode() > 204 || response.getStatusCode() < 200) { - importLogger.error("Error importing {} to {}, received {}", f.getAbsolutePath(), uri, + final String message = "Error while importing " + descriptionPath + " (" + + response.getStatusCode() + "): " + IOUtils.toString(response.getBody()); + logger.error(message); + importLogger.error("Error importing {} to {}, received {}", descriptionPath, destinationUri, response.getStatusCode()); - throw new RuntimeException("Error while importing membership resource " + f.getAbsolutePath() - + " (" + response.getStatusCode() + "): " + IOUtils.toString(response.getBody())); } else { - logger.info("Imported membership resource {}: {}", f.getAbsolutePath(), uri); - importLogger.info("import {} to {}", f.getAbsolutePath(), uri); + logger.info("Imported {}: {}", descriptionPath, destinationUri); + importLogger.info("import {} to {}", descriptionPath, destinationUri); successCount.incrementAndGet(); } - } catch (FcrepoOperationFailedException ex) { - importLogger.error( - String.format("Error importing: {} to {}, Message: {}", f.getAbsolutePath(), uri, ex.getMessage()), ex); - throw new RuntimeException("Error importing " + f.getAbsolutePath() + ": " + ex.toString(), ex); - } catch (IOException ex) { - importLogger.error( - String.format("Error reading/parsing file: {}, Message: {}", f.getAbsolutePath(), ex.getMessage()), ex); + } catch (final FcrepoOperationFailedException e) { + importLogger.error(String.format("Error importing {} to {}, Message: {}", + descriptionPath, destinationUri, e.getMessage()), e); + throw new RuntimeException( + "Error importing " + descriptionPath + ": " + e.toString(), e); + } catch (final IOException e) { + importLogger.error(String.format("Error reading/parsing {} to {}, Message: {}", + descriptionPath, destinationUri, e.getMessage()), e); throw new RuntimeException( - "Error reading or parsing " + f.getAbsolutePath() + ": " + ex.toString(), ex); + "Error reading or parsing " + descriptionPath + ": " + e.toString(), e); } } - private void importDirectory(final File dir) { - // process all the files first (because otherwise they might be - // created as peartree nodes which can't be updated with properties - // later. - if (dir.listFiles() != null) { - stream(dir.listFiles()).filter(File::isFile).forEach(file -> importFile(file)); - stream(dir.listFiles()).filter(File::isDirectory).forEach(directory -> importDirectory(directory)); + private PutBuilder descriptionBuilder(final ImportResource resc) throws FcrepoOperationFailedException, + IOException { + PutBuilder builder = client().put(resc.getDescriptionUri()) + .body(modelToStream(sanitize(resc)), config.getRdfLanguage()) + .ifUnmodifiedSince(currentTimestamp()); + if (sha1FileMap != null && config.getBagProfile() != null) { + // Use the bagIt checksum + final File containerFile = Paths.get(fileForContainerURI(resc.getUri()).toURI()).normalize().toFile(); + final String checksum = sha1FileMap.get(containerFile.getAbsolutePath()); + logger.debug("Using Bagit checksum ({}) for file ({})", checksum, containerFile.getPath()); + builder = builder.digest(checksum); } + return builder; } - private void importFile(final File f) { - // The path, relative to the base in the export directory. - // This is used in place of the full path to make the output more readable. - final String sourceRelativePath = - config.getBaseDirectory().toPath().relativize(f.toPath()).toString(); - final String filePath = f.getPath(); - if (filePath.endsWith(BINARY_EXTENSION) || filePath.endsWith(EXTERNAL_RESOURCE_EXTENSION)) { - // ... this is only expected to happen when binaries and metadata are written to the same directory... - if (config.isIncludeBinaries()) { - logger.debug("Skipping binary {}: it will be imported when its metadata is imported.", - sourceRelativePath); - } else { - logger.debug("Skipping binary {}", sourceRelativePath); - } - return; - } else if (!filePath.endsWith(config.getRdfExtension())) { - // this could be hidden files created by the OS - logger.info("Skipping file with unexpected extension ({}).", sourceRelativePath); - return; - } else { - - FcrepoResponse response = null; - URI destinationUri = null; - try { - final Model model = parseStream(new FileInputStream(f)); - // remove the member resources that are being imported - for (final ResIterator it = model.listSubjects(); it.hasNext();) { - final URI uri = URI.create(it.next().toString()); - if (relatedResources.contains(uri)) { - relatedResources.remove(uri); - } - } - - final ResIterator binaryResources = model.listResourcesWithProperty(RDF_TYPE, NON_RDF_SOURCE); - if (binaryResources.hasNext()) { - if (!config.isIncludeBinaries()) { - return; - } - destinationUri = new URI(binaryResources.nextResource().getURI()); - logger.info("Importing binary {}", sourceRelativePath); - response = importBinary(destinationUri, model); - } else { - destinationUri = uriForFile(f); - if (membershipResources.contains(destinationUri)) { - logger.warn("Skipping Membership Resource: {}", destinationUri); - return; - } - if (model.contains(null, RDF_TYPE, PAIRTREE)) { - logger.info("Skipping PairTree Resource: {}", destinationUri); - return; - } - - logger.info("Importing container {} to {}", f.getAbsolutePath(), destinationUri); - response = importContainer(destinationUri, sanitize(model)); - } + private void importBinaryFile(final ImportResource resc) throws FcrepoOperationFailedException, IOException { + final URI binaryURI = resc.getMappedUri(); + final Model model = resc.getModel(); + final String contentType = model + .getProperty(createResource(binaryURI.toString()), HAS_MIME_TYPE).getString(); + final File binaryFile = resc.getBinary(); + FcrepoResponse binaryResponse; + binaryResponse = binaryBuilder(binaryURI, binaryFile, contentType, model).perform(); - if (response == null) { - logger.warn("Failed to import {}", f.getAbsolutePath()); - } else if (response.getStatusCode() == 401) { - importLogger.error("Error importing {} to {}, 401 Unauthorized", f.getAbsolutePath(), - destinationUri); - throw new AuthenticationRequiredRuntimeException(); - } else if (response.getStatusCode() > 204 || response.getStatusCode() < 200) { - final String message = "Error while importing " + f.getAbsolutePath() + " (" - + response.getStatusCode() + "): " + IOUtils.toString(response.getBody()); - logger.error(message); - importLogger.error("Error importing {} to {}, received {}", f.getAbsolutePath(), destinationUri, - response.getStatusCode()); - } else { - logger.info("Imported {}: {}", f.getAbsolutePath(), destinationUri); - importLogger.info("import {} to {}", f.getAbsolutePath(), destinationUri); - successCount.incrementAndGet(); - } - } catch (FcrepoOperationFailedException ex) { - importLogger.error(String.format("Error importing {} to {}, Message: {}", f.getAbsolutePath(), - destinationUri, ex.getMessage()), ex); - throw new RuntimeException("Error importing " + f.getAbsolutePath() + ": " + ex.toString(), ex); - } catch (IOException ex) { - importLogger.error(String.format("Error reading/parsing {} to {}, Message: {}", f.getAbsolutePath(), - destinationUri, ex.getMessage()), ex); - throw new RuntimeException( - "Error reading or parsing " + f.getAbsolutePath() + ": " + ex.toString(), ex); - } catch (URISyntaxException ex) { - importLogger.error( - String.format("Error building URI for {}, Message: {}", f.getAbsolutePath(), ex.getMessage()), ex); - throw new RuntimeException("Error building URI for " + f.getAbsolutePath() + ": " + ex.toString(), ex); - } + if (binaryResponse.getStatusCode() == 410 && config.overwriteTombstones()) { + throw new ResourceGoneRuntimeException(binaryResponse); } - } - private Model parseStream(final InputStream in) throws IOException { - final SubjectMappingStreamRDF mapper = new SubjectMappingStreamRDF(config.getSource(), - config.getDestination()); - try (final InputStream in2 = in) { - RDFDataMgr.parse(mapper, in2, contentTypeToLang(config.getRdfLanguage())); - } - return mapper.getModel(); - } - - private FcrepoResponse importBinary(final URI binaryURI, final Model model) - throws FcrepoOperationFailedException, IOException { - final String contentType = model.getProperty(createResource(binaryURI.toString()), HAS_MIME_TYPE).getString(); - final File binaryFile = fileForBinaryURI(binaryURI, external(contentType)); - final FcrepoResponse binaryResponse = binaryBuilder(binaryURI, binaryFile, contentType, model).perform(); + // Check for success importing file if (binaryResponse.getStatusCode() == 201 || binaryResponse.getStatusCode() == 204) { logger.info("Imported binary: {}", binaryURI); importLogger.info("import {} to {}", binaryFile.getAbsolutePath(), binaryURI); successCount.incrementAndGet(); - - final URI descriptionURI = binaryResponse.getLinkHeaders("describedby").get(0); - return client().put(descriptionURI).body(modelToStream(sanitize(model)), config.getRdfLanguage()) - .preferLenient().perform(); - } else if (binaryResponse.getStatusCode() == 410 && config.overwriteTombstones()) { - deleteTombstone(binaryResponse); - return binaryBuilder(binaryURI, binaryFile, contentType, model).perform(); } else { - logger.error("Error while importing {} ({}): {}", binaryFile.getAbsolutePath(), + // Failed to import file, throw exception + final String message = String.format("Error while importing %s (%s): %s", binaryFile.getAbsolutePath(), binaryResponse.getStatusCode(), IOUtils.toString(binaryResponse.getBody())); - return null; + throw new BinaryImportException(message); } } @@ -476,49 +387,16 @@ private PutBuilder binaryBuilder(final URI binaryURI, final File binaryFile, fin return builder; } - private boolean external(final String contentType) { - return contentType.startsWith("message/external-body"); - } - - private FcrepoResponse importContainer(final URI uri, final Model model) throws FcrepoOperationFailedException { - final FcrepoResponse response = containerBuilder(uri, model).preferLenient().perform(); - if (response.getStatusCode() == 410 && config.overwriteTombstones()) { - deleteTombstone(response); - return containerBuilder(uri, model).preferLenient().perform(); - } else { - return response; - } - } - - private PutBuilder containerBuilder(final URI uri, final Model model) throws FcrepoOperationFailedException { - PutBuilder builder = client().put(uri) - .body(modelToStream(model), config.getRdfLanguage()) - .ifUnmodifiedSince(currentTimestamp()); - if (sha1FileMap != null && config.getBagProfile() != null) { - // Use the bagIt checksum - final File containerFile = Paths.get(fileForContainerURI(uri).toURI()).normalize().toFile(); - final String checksum = sha1FileMap.get(containerFile.getAbsolutePath()); - logger.debug("Using Bagit checksum ({}) for file ({})", checksum, containerFile.getPath()); - builder = builder.digest(checksum); - } - return builder; - } - private String currentTimestamp() { return RFC_1123_DATE_TIME.format(ZonedDateTime.now(ZoneId.of("GMT"))); } - private void deleteTombstone(final FcrepoResponse response) throws FcrepoOperationFailedException { - final URI tombstone = response.getLinkHeaders("hasTombstone").get(0); + private void deleteTombstone(final URI rescUri, final URI tombstone) throws FcrepoOperationFailedException { if (tombstone != null) { client().delete(tombstone).perform(); } else { - String uri = response.getUrl().toString(); - if (uri.endsWith("/")) { - uri = uri.substring(0, uri.length() - 1); - } - final URI parent = URI.create(uri.substring(0, uri.lastIndexOf("/", uri.length() - 1))); - deleteTombstone(client().head(parent).perform()); + final FcrepoResponse response = client().head(rescUri).perform(); + deleteTombstone(rescUri, response.getLinkHeaders("hasTombstone").get(0)); } } @@ -539,7 +417,11 @@ private void deleteTombstone(final FcrepoResponse response) throws FcrepoOperati * @throws IOException * @throws FcrepoOperationFailedException */ - private Model sanitize(final Model model) throws IOException, FcrepoOperationFailedException { + private Model sanitize(final ImportResource resc) throws IOException, FcrepoOperationFailedException { + + final Set membershipRelations = uriToMembershipRelations.get(resc.getUri().toString()); + + final Model model = resc.getModel(); final List remove = new ArrayList<>(); for (final StmtIterator it = model.listStatements(); it.hasNext(); ) { final Statement s = it.nextStatement(); @@ -554,10 +436,18 @@ private Model sanitize(final Model model) throws IOException, FcrepoOperationFai || (s.getPredicate().equals(RDF_TYPE) && forbiddenType(s.getResource()))) { remove.add(s); } else if (s.getObject().isResource()) { - // make sure that referenced repository objects exist final String obj = s.getResource().toString(); + if (obj.startsWith(repositoryRoot.toString())) { - ensureExists(URI.create(obj)); + if (membershipRelations != null && membershipRelations.contains(s.getPredicate().getURI())) { + // trim out generated membership relations + remove.add(s); + } else { + // make sure that referenced repository objects exist + ensureExists(URI.create(obj)); + } + + } } } @@ -633,31 +523,6 @@ private InputStream modelToStream(final Model model) { return new ByteArrayInputStream(buf.toByteArray()); } - private URI uriForFile(final File f) { - // get path of file relative to the data directory - String relative = config.getBaseDirectory().toURI().relativize(f.toURI()).toString(); - relative = TransferProcess.decodePath(relative); - - // rebase the path on the destination uri (translating source/destination if needed) - if ( config.getSource() != null && config.getDestination() != null ) { - relative = baseURI(config.getSource()) + relative; - relative = relative.replaceFirst(config.getSource().toString(), config.getDestination().toString()); - } else { - relative = baseURI(config.getResource()) + relative; - } - - // for exported RDF, just remove the ".extension" and you have the encoded path - if (relative.endsWith(config.getRdfExtension())) { - relative = relative.substring(0, relative.length() - config.getRdfExtension().length()); - } - return URI.create(relative); - } - - private static String baseURI(final URI uri) { - final String base = uri.toString().replaceFirst(uri.getPath() + "$", ""); - return (base.endsWith("/")) ? base : base + "/"; - } - private File fileForBinaryURI(final URI uri, final boolean external) { if (external) { return fileForExternalResources(uri, config.getSourcePath(), config.getDestinationPath(), @@ -678,6 +543,10 @@ private File directoryForContainer(final URI uri) { config.getDestinationPath(), config.getBaseDirectory()); } + private boolean external(final String contentType) { + return contentType.startsWith("message/external-body"); + } + /** * Verify the bag we are going to import * @@ -689,7 +558,7 @@ public static boolean verifyBag(final File bagDir) { final Bag bag = BagReader.read(bagDir); BagVerifier.isValid(bag, true); return true; - } catch (Exception e) { + } catch (final Exception e) { throw new RuntimeException(String.format("Error verifying bag: %s", e.getMessage()), e); } } @@ -713,7 +582,7 @@ public URI findRepositoryRoot(final URI uri) { return findRepositoryRoot(URI.create(u.toString().substring(0, u.toString().lastIndexOf("/")))); } - } catch (ResourceNotFoundRuntimeException ex) { + } catch (final ResourceNotFoundRuntimeException ex) { // The targeted resource that need to be imported next return findRepositoryRoot(URI.create(u.toString().substring(0, u.toString().lastIndexOf("/")))); @@ -723,4 +592,44 @@ public URI findRepositoryRoot(final URI uri) { throw new RuntimeException("Error finding repository root " + u, ex); } } + + /** + * Finds containers which are providing membership relations, then uses that information to track which membership + * relations need to be removed before import. + * + * @param importDirectory directory containing the import + * @throws IOException Thrown if the import directory cannot be read + */ + private void discoverMembershipResources(final File importDirectory) throws IOException { + Files.walkFileTree(importDirectory.toPath(), new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + if (!file.toString().endsWith(config.getRdfExtension())) { + return CONTINUE; + } + + final Model model = mapRdfStream(new FileInputStream(file.toFile()), config); + if (model.contains(null, MEMBERSHIP_RESOURCE, (RDFNode) null)) { + model.listObjectsOfProperty(MEMBERSHIP_RESOURCE).forEachRemaining(node -> { + logger.info("Membership resource: {}", node); + final String uri = node.toString(); + + final Set relations; + if (uriToMembershipRelations.containsKey(uri)) { + relations = uriToMembershipRelations.get(uri); + } else { + relations = new HashSet<>(); + uriToMembershipRelations.put(uri, relations); + } + + model.listObjectsOfProperty(HAS_MEMBER_RELATION).forEachRemaining(relNode -> { + relations.add(relNode.toString()); + }); + }); + } + + return CONTINUE; + } + }); + } } diff --git a/src/main/java/org/fcrepo/importexport/importer/SubjectMappingStreamRDF.java b/src/main/java/org/fcrepo/importexport/importer/SubjectMappingStreamRDF.java index a68252b5..40667e84 100644 --- a/src/main/java/org/fcrepo/importexport/importer/SubjectMappingStreamRDF.java +++ b/src/main/java/org/fcrepo/importexport/importer/SubjectMappingStreamRDF.java @@ -18,21 +18,20 @@ package org.fcrepo.importexport.importer; import static org.apache.jena.graph.Factory.createDefaultGraph; -import static org.apache.jena.rdf.model.ModelFactory.createModelForGraph; import static org.apache.jena.graph.NodeFactory.createURI; +import static org.apache.jena.rdf.model.ModelFactory.createModelForGraph; import static org.slf4j.LoggerFactory.getLogger; import java.net.URI; -import org.apache.jena.rdf.model.Model; -import org.slf4j.Logger; - import org.apache.jena.atlas.lib.Sink; import org.apache.jena.graph.Graph; import org.apache.jena.graph.Node; import org.apache.jena.graph.Triple; +import org.apache.jena.rdf.model.Model; import org.apache.jena.riot.lang.SinkTriplesToGraph; import org.apache.jena.riot.system.StreamRDFBase; +import org.slf4j.Logger; /** * StreamRDF implementation that maps URIs to a specified destination URI. @@ -43,8 +42,8 @@ public class SubjectMappingStreamRDF extends StreamRDFBase { private static final Logger logger = getLogger(SubjectMappingStreamRDF.class); - private final String sourceURI; - private final String destinationURI; + protected final String sourceURI; + protected final String destinationURI; private final Graph graph; private final Sink sink; @@ -65,7 +64,7 @@ public void triple(final Triple t) { sink.send(Triple.create(rebase(t.getSubject()), t.getPredicate(), rebase(t.getObject()))); } - private Node rebase(final Node node) { + protected Node rebase(final Node node) { if (node.isURI() && sourceURI != null && destinationURI != null && node.getURI().startsWith(sourceURI)) { return createURI(node.getURI().replaceFirst(sourceURI, destinationURI)); diff --git a/src/main/java/org/fcrepo/importexport/importer/VersionDiffDeletionGenerator.java b/src/main/java/org/fcrepo/importexport/importer/VersionDiffDeletionGenerator.java new file mode 100644 index 00000000..1ff4e095 --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/importer/VersionDiffDeletionGenerator.java @@ -0,0 +1,208 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static org.fcrepo.importexport.common.FcrepoConstants.CREATED_DATE; +import static org.fcrepo.importexport.common.FcrepoConstants.FCR_VERSIONS_PATH; +import static org.fcrepo.importexport.common.TransferProcess.directoryForContainer; +import static org.fcrepo.importexport.common.URITranslationUtil.remapResourceUri; +import static org.fcrepo.importexport.common.UriUtils.withSlash; +import static org.slf4j.LoggerFactory.getLogger; + +import java.io.File; +import java.net.URI; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.DirectoryFileFilter; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.jena.datatypes.xsd.XSDDateTime; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ResIterator; +import org.apache.jena.rdf.model.Resource; +import org.fcrepo.importexport.common.Config; +import org.fcrepo.importexport.common.FcrepoConstants; +import org.slf4j.Logger; + +/** + * Generates events to cause the cleanup of resources that are deleted between versions of a resource tree. + * + * @author bbpennel + */ +public class VersionDiffDeletionGenerator { + private static final Logger logger = getLogger(VersionDiffDeletionGenerator.class); + + private static final String FCR_VERSIONS_ESCAPED = "fcr%3Aversions"; + + final Config config; + + /** + * Constructs a VersionDeletionGenerator + * + * @param config config + */ + public VersionDiffDeletionGenerator(final Config config) { + this.config = config; + } + + /** + * Gets a list of import deletion events from the given fcr:versions model + * + * @param versionsModel fcr:versions model + * @return list of deletion events + */ + public List generateImportDeletions(final Model versionsModel) { + final List timestampedUris = getTimestampSortedVersionUris(versionsModel); + + final Map> versionFileMap = getFilesPerVersion(timestampedUris); + + return makeDeletionListFromFileDiffs(timestampedUris, versionFileMap); + } + + private List getTimestampSortedVersionUris(final Model versionsModel) { + final List timestampedUris = new ArrayList<>(); + + final ResIterator vRescIt = versionsModel.listResourcesWithProperty(CREATED_DATE); + while (vRescIt.hasNext()) { + final Resource vResc = vRescIt.next(); + final URI rescUri = URI.create(vResc.getURI()); + final XSDDateTime dateTime = (XSDDateTime) vResc.getProperty(CREATED_DATE).getLiteral().getValue(); + final long time = dateTime.asCalendar().getTimeInMillis(); + + timestampedUris.add(new TimestampUriPair(time, rescUri)); + } + + // Sort the version creation events + timestampedUris.sort(new Comparator() { + @Override + public int compare(final TimestampUriPair o1, final TimestampUriPair o2) { + return new Long(o1.timestamp).compareTo(o2.timestamp); + } + }); + + // Set head timestamp for deletion to just after the most recent version was created + final long headTimestamp = timestampedUris.get(timestampedUris.size() - 1).timestamp + 1; + + final String headVersionUri = versionsModel + .listResourcesWithProperty(FcrepoConstants.HAS_VERSION).next().getURI(); + + timestampedUris.add(new TimestampUriPair(headTimestamp, URI.create(headVersionUri))); + + return timestampedUris; + } + + private Map> getFilesPerVersion(final List timestampedUris) { + final Map> versionFileMap = new HashMap<>(); + timestampedUris.forEach(timestampedUriPair -> { + final File versionDir = getVersionDirectory(timestampedUriPair.uri); + final Path versionPath = versionDir.toPath(); + + // No child resources, return empty set of file paths + if (!versionDir.exists() || !versionDir.isDirectory()) { + versionFileMap.put(timestampedUriPair.uri, new HashSet<>()); + return; + } + + final boolean excludeVersionDir = !timestampedUriPair.uri.toString().contains(FCR_VERSIONS_PATH); + + final Set relativeVersionFiles = FileUtils.listFiles(versionDir, + new IOFileFilter() { + @Override + public boolean accept(final File file) { + final String absPath = file.toPath().toString(); + if (excludeVersionDir && absPath.contains(FCR_VERSIONS_ESCAPED)) { + return false; + } + return absPath.endsWith(config.getRdfExtension()); + } + + @Override + public boolean accept(final File dir, final String name) { + return false; + } + }, + DirectoryFileFilter.DIRECTORY) + .stream().map(f -> { + return versionPath.relativize(f.toPath()).toString(); + }).collect(Collectors.toCollection(HashSet::new)); + + versionFileMap.put(timestampedUriPair.uri, relativeVersionFiles); + }); + + return versionFileMap; + } + + private List makeDeletionListFromFileDiffs(final List timestampedUris, + final Map> versionFileMap) { + final String fcrMetadataEscaped = "fcr%3Ametadata" + config.getRdfExtension(); + + final List deletions = new ArrayList<>(); + + for (int i = 1; i < timestampedUris.size(); i++) { + final TimestampUriPair previousVersion = timestampedUris.get(i - 1); + final TimestampUriPair currentVersion = timestampedUris.get(i); + + final Set previousFiles = versionFileMap.get(previousVersion.uri); + final Set currentFiles = versionFileMap.get(currentVersion.uri); + + previousFiles.removeAll(currentFiles); + previousFiles.forEach(relativeFile -> { + String relativePath = relativeFile; + if (relativeFile.endsWith(fcrMetadataEscaped)) { + relativePath = relativeFile.substring(0, relativeFile.length() - fcrMetadataEscaped.length() - 1); + } else if (relativeFile.endsWith(config.getRdfExtension())) { + relativePath = relativeFile.substring(0, relativeFile.length() - config.getRdfExtension().length()); + } + + final URI fileUri = URI.create(previousVersion.uri.toString() + "/" + relativePath); + final URI remappedUri = remapResourceUri(fileUri, + config.getSource(), config.getDestination()); + + deletions.add(new ImportDeletion(remappedUri, currentVersion.timestamp, config)); + + logger.debug("Adding version deletion of {}", remappedUri); + }); + + } + + return deletions; + } + + private File getVersionDirectory(final URI uri) { + return directoryForContainer(withSlash(uri), config.getSourcePath(), config.getDestinationPath(), + config.getBaseDirectory()); + } + + private class TimestampUriPair { + public final long timestamp; + public final URI uri; + + public TimestampUriPair(final long timestamp, final URI uri) { + this.timestamp = timestamp; + this.uri = uri; + } + } +} diff --git a/src/main/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDF.java b/src/main/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDF.java new file mode 100644 index 00000000..800f8bb3 --- /dev/null +++ b/src/main/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDF.java @@ -0,0 +1,58 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static org.apache.jena.graph.NodeFactory.createURI; +import static org.fcrepo.importexport.common.URITranslationUtil.remapResourceUri; + +import java.net.URI; + +import org.apache.jena.graph.Node; + +/** + * StreamRDF implementation that maps URIs to a specified destination URI and strips out version paths. + * + * @author escowles + * @author bbpennel + * + */ +public class VersionSubjectMappingStreamRDF extends SubjectMappingStreamRDF { + + /** + * Create a version subject-mapping RDF stream + * @param sourceURI the source URI to map triples from + * @param destinationURI the destination URI to map triples to + */ + public VersionSubjectMappingStreamRDF(final URI sourceURI, final URI destinationURI) { + super(sourceURI, destinationURI); + } + + @Override + protected Node rebase(final Node node) { + if (node.isURI()) { + // Replace uri base and strip out versions path + final String rebasedNodeUri = remapResourceUri(node.getURI(), sourceURI, destinationURI); + + if (!rebasedNodeUri.equals(node.getURI())) { + return createURI(rebasedNodeUri); + } + } + + return node; + } +} diff --git a/src/test/java/org/fcrepo/importexport/exporter/ExportVersionsTest.java b/src/test/java/org/fcrepo/importexport/exporter/ExportVersionsTest.java index 9ce61391..2587ecbb 100644 --- a/src/test/java/org/fcrepo/importexport/exporter/ExportVersionsTest.java +++ b/src/test/java/org/fcrepo/importexport/exporter/ExportVersionsTest.java @@ -20,13 +20,13 @@ import static java.util.Collections.emptyList; import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINER; import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; -import static org.fcrepo.importexport.common.FcrepoConstants.CREATED_DATE; import static org.fcrepo.importexport.common.FcrepoConstants.FCR_VERSIONS_PATH; -import static org.fcrepo.importexport.common.FcrepoConstants.HAS_VERSION; -import static org.fcrepo.importexport.common.FcrepoConstants.HAS_VERSION_LABEL; import static org.fcrepo.importexport.common.FcrepoConstants.NON_RDF_SOURCE; import static org.fcrepo.importexport.common.FcrepoConstants.RDF_SOURCE; import static org.fcrepo.importexport.common.FcrepoConstants.REPOSITORY_ROOT; +import static org.fcrepo.importexport.test.util.JsonLdResponse.addVersionJson; +import static org.fcrepo.importexport.test.util.JsonLdResponse.createJson; +import static org.fcrepo.importexport.test.util.JsonLdResponse.joinJsonArray; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.isA; @@ -41,11 +41,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.http.HttpStatus; -import org.apache.jena.rdf.model.Resource; import org.fcrepo.client.FcrepoClient; import org.fcrepo.client.FcrepoOperationFailedException; import org.fcrepo.client.FcrepoResponse; @@ -329,43 +327,4 @@ public void testExportBinaryFromRepositoryRoot() throws Exception { assertTrue(exporter.wroteFile(new File(basedir + "/rest/file1/fcr%3Aversions/version1.binary"))); assertTrue(exporter.wroteFile(new File(basedir + "/rest/file1/fcr%3Aversions/version1/fcr%3Ametadata.jsonld"))); } - - private String createJson(final URI resource, final URI... children) { - return createJson(resource, null, children); - } - - private String createJson(final URI resource, final Resource type, final URI... children) { - final StringBuilder json = new StringBuilder("{\"@id\":\"" + resource.toString() + "\""); - if (type != null) { - json.append(",\"@type\":[\"" + type.getURI() + "\"]"); - } - if (children != null && children.length > 0) { - json.append(",\"" + CONTAINS.getURI() + "\":[") - .append(Arrays.stream(children) - .map(child -> "{\"@id\":\"" + child.toString() + "\"}") - .collect(Collectors.joining(","))) - .append(']'); - } - json.append('}'); - return json.toString(); - } - - private String joinJsonArray(final List array) { - return "[" + String.join(",", array) + "]"; - } - - private List addVersionJson(final List versions, final URI rescUri, final URI versionUri, - final String label, final String timestamp) { - final String versionJson = "{\"@id\":\"" + rescUri.toString() + "\"," + - "\"" + HAS_VERSION.getURI() + "\":[{\"@id\":\"" + versionUri.toString() + "\"}]}," + - "{\"@id\":\"" + versionUri.toString() + "\"," + - "\"" + CREATED_DATE.getURI() + "\":[{" + - "\"@value\":\"" + timestamp + "\"," + - "\"@type\": \"http://www.w3.org/2001/XMLSchema#dateTime\"}]," + - "\"" + HAS_VERSION_LABEL.getURI() + "\":[{" + - "\"@value\":\"" + label + " \"}]" + - "}"; - versions.add(versionJson); - return versions; - } } diff --git a/src/test/java/org/fcrepo/importexport/importer/ChronologicalImportEventIteratorTest.java b/src/test/java/org/fcrepo/importexport/importer/ChronologicalImportEventIteratorTest.java new file mode 100644 index 00000000..0e77e440 --- /dev/null +++ b/src/test/java/org/fcrepo/importexport/importer/ChronologicalImportEventIteratorTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.fcrepo.importexport.importer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import java.io.File; +import java.net.URI; + +import org.fcrepo.importexport.common.Config; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +/** + * @author bbpennel + */ +public class ChronologicalImportEventIteratorTest { + + private final static URI restUri = URI.create("http://localhost:8080/rest"); + + private ChronologicalImportEventIterator rescIt; + + @Mock + private Config config; + + @Mock + private ImportResource impResource; + + @Mock + private ImportResource restResource; + + @Before + public void setup() throws Exception { + initMocks(this); + + when(config.getRdfExtension()).thenReturn(".jsonld"); + when(config.getRdfLanguage()).thenReturn("application/ld+json"); + when(config.getResource()).thenReturn(restUri); + + when(restResource.getUri()).thenReturn(restUri); + } + + @Test + public void testSingleContainer() throws Exception { + final URI resourceUri = new URI("http://localhost:8080/rest/con1"); + + final File directory = new File("src/test/resources/sample/container"); + when(config.getBaseDirectory()).thenReturn(directory); + + rescIt = new ChronologicalImportEventIterator(directory, config); + + assertTrue(rescIt.hasNext()); + final ImportEvent resc = rescIt.next(); + assertEquals(resourceUri, resc.getUri()); + assertFalse(rescIt.hasNext()); + } + + @Test + public void testBinary() throws Exception { + final URI resourceUri = new URI("http://localhost:8080/rest/bin1"); + + final File directory = new File("src/test/resources/sample/binary"); + when(config.getBaseDirectory()).thenReturn(directory); + + rescIt = new ChronologicalImportEventIterator(directory, config); + + assertTrue(rescIt.hasNext()); + + final ImportEvent resc1 = rescIt.next(); + assertEquals("http://localhost:8080/rest", resc1.getUri().toString()); + final ImportEvent resc2 = rescIt.next(); + assertEquals(resourceUri, resc2.getUri()); + + assertFalse(rescIt.hasNext()); + } + + @Test + public void testVersions() throws Exception { + when(config.includeVersions()).thenReturn(true); + + final URI con1Uri = new URI("http://localhost:8080/fcrepo/rest/con1"); + final URI con1VOriginalUri = new URI( + "http://localhost:8080/fcrepo/rest/con1/fcr:versions/version_original"); + final URI con1V1Uri = new URI( + "http://localhost:8080/fcrepo/rest/con1/fcr:versions/version_1"); + final URI con2Uri = new URI( + "http://localhost:8080/fcrepo/rest/con1/con2"); + final URI con2VOriginalUri = new URI( + "http://localhost:8080/fcrepo/rest/con1/fcr:versions/version_original/con2"); + final URI bin1Uri = new URI("http://localhost:8080/fcrepo/rest/con1/bin1"); + final URI bin1V1Uri = new URI( + "http://localhost:8080/fcrepo/rest/con1/fcr:versions/version_1/bin1"); + + final File directory = new File("src/test/resources/sample/versioned"); + when(config.getBaseDirectory()).thenReturn(directory); + + rescIt = new ChronologicalImportEventIterator(directory, config); + + assertTrue(rescIt.hasNext()); + + assertEquals(con1VOriginalUri, rescIt.next().getUri()); + assertEquals(con2VOriginalUri, rescIt.next().getUri()); + final ImportEvent version1 = rescIt.next(); + assertTrue("Expected version creation event", version1 instanceof ImportVersion); + assertEquals(con1VOriginalUri, version1.getUri()); + + assertEquals(con1V1Uri, rescIt.next().getUri()); + assertEquals(bin1V1Uri, rescIt.next().getUri()); + + final ImportEvent deleteCon2 = rescIt.next(); + assertTrue(deleteCon2 instanceof ImportDeletion); + assertEquals(con2Uri, deleteCon2.getUri()); + + final ImportEvent version2 = rescIt.next(); + assertTrue("Expected version creation event", version2 instanceof ImportVersion); + assertEquals(con1V1Uri, version2.getUri()); + + assertEquals(con1Uri, rescIt.next().getUri()); + assertEquals(bin1Uri, rescIt.next().getUri()); + + assertFalse(rescIt.hasNext()); + } + + @Test + public void testExcludeVersions() throws Exception { + when(config.includeVersions()).thenReturn(false); + + final URI con1Uri = new URI("http://localhost:8080/fcrepo/rest/con1"); + final URI bin1Uri = new URI("http://localhost:8080/fcrepo/rest/con1/bin1"); + + final File directory = new File("src/test/resources/sample/versioned"); + when(config.getBaseDirectory()).thenReturn(directory); + + rescIt = new ChronologicalImportEventIterator(directory, config); + + assertTrue(rescIt.hasNext()); + + assertEquals(con1Uri, rescIt.next().getUri()); + assertEquals(bin1Uri, rescIt.next().getUri()); + + assertFalse(rescIt.hasNext()); + } + + @Test + public void testUnmodifiedVersions() throws Exception { + when(config.includeVersions()).thenReturn(true); + + final File directory = new File("src/test/resources/sample/versioning/unmodified"); + when(config.getBaseDirectory()).thenReturn(directory); + + final URI con1Uri = new URI( + "http://localhost:8080/rest/v_con1"); + final URI con1VOriginalUri = new URI( + "http://localhost:8080/rest/v_con1/fcr:versions/version_original"); + final URI con1VUnmodUri = new URI( + "http://localhost:8080/rest/v_con1/fcr:versions/version_unchanged"); + final URI child1Uri = new URI("http://localhost:8080/rest/v_con1/child1"); + + rescIt = new ChronologicalImportEventIterator(directory, config); + + assertTrue(rescIt.hasNext()); + + assertEquals(con1Uri, rescIt.next().getUri()); + // the two versions of the child are identical, so it is undefined which of the two will be returned + assertTrue(rescIt.next().getUri().toString().matches(".*v_con1/fcr:versions/.*/child1")); + + final ImportEvent versionOriginal = rescIt.next(); + assertTrue("Expected version creation event", versionOriginal instanceof ImportVersion); + assertEquals(con1VOriginalUri, versionOriginal.getUri()); + + final ImportEvent versionUnmod = rescIt.next(); + assertTrue("Expected version creation event", versionUnmod instanceof ImportVersion); + assertEquals(con1VUnmodUri, versionUnmod.getUri()); + + assertEquals(child1Uri, rescIt.next().getUri()); + + assertFalse(rescIt.hasNext()); + } + + @Test + public void testStartingDirectoryDifferentFromBase() throws Exception { + when(config.getRdfExtension()).thenReturn(".ttl"); + when(config.getRdfLanguage()).thenReturn("text/turtle"); + + final File baseDirectory = new File("src/test/resources/sample/mapped"); + when(config.getBaseDirectory()).thenReturn(baseDirectory); + + final File startingDirectory = new File("src/test/resources/sample/mapped/rest/dev/"); + + final URI con1Uri = new URI("http://localhost:8080/rest/dev/asdf"); + + rescIt = new ChronologicalImportEventIterator(startingDirectory, config); + + assertTrue(rescIt.hasNext()); + + assertEquals(con1Uri, rescIt.next().getUri()); + + assertFalse(rescIt.hasNext()); + } + + @Test + public void testParentUpdatedAfterChild() throws Exception { + when(config.includeVersions()).thenReturn(false); + + final File baseDirectory = new File("src/test/resources/sample/recent_parent"); + when(config.getBaseDirectory()).thenReturn(baseDirectory); + + final URI con1Uri = new URI("http://localhost:8080/rest/con1"); + final URI con2Uri = new URI("http://localhost:8080/rest/con1/con2"); + + rescIt = new ChronologicalImportEventIterator(baseDirectory, config); + + assertTrue(rescIt.hasNext()); + + assertEquals(con1Uri, rescIt.next().getUri()); + assertEquals(con2Uri, rescIt.next().getUri()); + + assertFalse(rescIt.hasNext()); + } + + @Test + public void testParentUpdatedAfterChildWithVersions() throws Exception { + when(config.includeVersions()).thenReturn(true); + + final File baseDirectory = new File("src/test/resources/sample/recent_parent"); + when(config.getBaseDirectory()).thenReturn(baseDirectory); + + final URI con1Uri = new URI("http://localhost:8080/rest/con1"); + final URI con2Uri = new URI("http://localhost:8080/rest/con1/con2"); + final URI con1V0Uri = new URI("http://localhost:8080/rest/con1/fcr:versions/v0"); + final URI con2V0Uri = new URI("http://localhost:8080/rest/con1/fcr:versions/v0/con2"); + final URI v0Uri = new URI("http://localhost:8080/rest/con1/fcr:versions/v0"); + + rescIt = new ChronologicalImportEventIterator(baseDirectory, config); + + assertTrue(rescIt.hasNext()); + + assertEquals(con1V0Uri, rescIt.next().getUri()); + assertEquals(con2V0Uri, rescIt.next().getUri()); + assertEquals(v0Uri, rescIt.next().getUri()); + assertEquals(con1Uri, rescIt.next().getUri()); + assertEquals(con2Uri, rescIt.next().getUri()); + + assertFalse(rescIt.hasNext()); + } +} diff --git a/src/test/java/org/fcrepo/importexport/importer/ImportVersionsTest.java b/src/test/java/org/fcrepo/importexport/importer/ImportVersionsTest.java new file mode 100644 index 00000000..e5281a45 --- /dev/null +++ b/src/test/java/org/fcrepo/importexport/importer/ImportVersionsTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINER; +import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; +import static org.fcrepo.importexport.common.FcrepoConstants.FCR_VERSIONS_PATH; +import static org.fcrepo.importexport.common.FcrepoConstants.NON_RDF_SOURCE; +import static org.fcrepo.importexport.common.FcrepoConstants.REPOSITORY_ROOT; +import static org.fcrepo.importexport.test.util.JsonLdResponse.createJson; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.isA; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.MockitoAnnotations.initMocks; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.fcrepo.client.FcrepoClient; +import org.fcrepo.client.FcrepoOperationFailedException; +import org.fcrepo.client.FcrepoResponse; +import org.fcrepo.client.HeadBuilder; +import org.fcrepo.client.PostBuilder; +import org.fcrepo.client.PutBuilder; +import org.fcrepo.importexport.common.Config; +import org.fcrepo.importexport.test.util.ResponseMocker; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +/** + * + * @author bbpennel + * + */ +public class ImportVersionsTest { + + private URI rootResource; + + @Mock + private FcrepoClient client; + @Mock + private FcrepoClient.FcrepoClientBuilder clientBuilder; + @Mock + private FcrepoResponse headResponse; + @Mock + private HeadBuilder headBuilder; + @Mock + private PutBuilder binBuilder; + @Mock + private PutBuilder containerBuilder; + + private Config config; + + private URI binaryURI; + private URI binaryMDURI; + private URI containerURI; + private URI container2URI; + + private String[] predicates = new String[]{ CONTAINS.toString() }; + private List containerLinks = + Arrays.asList(URI.create(CONTAINER.getURI())); + private List binaryLinks = + Arrays.asList(URI.create(NON_RDF_SOURCE.getURI())); + + @Before + public void setUp() throws Exception { + initMocks(this); + + containerURI = new URI("http://localhost:8080/fcrepo/rest/con1"); + container2URI = new URI("http://localhost:8080/fcrepo/rest/con1/con2"); + binaryURI = new URI("http://localhost:8080/fcrepo/rest/con1/bin1"); + binaryMDURI = new URI("http://localhost:8080/fcrepo/rest/con1/bin1/fcr:metadata"); + + when(clientBuilder.build()).thenReturn(client); + + rootResource = new URI("http://localhost:8080/rest"); + mockResponse(rootResource, containerLinks, new ArrayList<>(), + createJson(rootResource, REPOSITORY_ROOT)); + + config = new Config(); + config.setMode("import"); + config.setIncludeVersions(true); + config.setIncludeBinaries(true); + config.setRdfLanguage("application/ld+json"); + config.setBaseDirectory("src/test/resources/sample/versioned"); + config.setResource(rootResource); + + final HeadBuilder headBuilder = mock(HeadBuilder.class); + when(client.head(isA(URI.class))).thenReturn(headBuilder); + when(headBuilder.disableRedirects()).thenReturn(headBuilder); + when(headBuilder.perform()).thenReturn(headResponse); + when(headResponse.getStatusCode()).thenReturn(200); + } + + private void mockResponse(final URI uri, final List typeLinks, final List describedbyLinks, + final String body) throws FcrepoOperationFailedException { + ResponseMocker.mockHeadResponse(client, uri, typeLinks, describedbyLinks); + + ResponseMocker.mockGetResponse(client, uri, typeLinks, describedbyLinks, body); + } + + @Test + public void testImportVersionsOff() throws Exception { + config.setIncludeVersions(false); + + final URI versionUri = URI.create(containerURI.toString() + "/" + FCR_VERSIONS_PATH); + + ResponseMocker.mockPutResponse(client, containerURI); + ResponseMocker.mockPutResponse(client, container2URI); + ResponseMocker.mockPutResponse(client, binaryURI); + ResponseMocker.mockPutResponse(client, binaryMDURI); + ResponseMocker.mockPostResponse(client, versionUri); + + final Importer importer = new Importer(config, clientBuilder); + importer.run(); + + // Container should only be updated for head version + verify(client).put(eq(containerURI)); + verify(client, never()).put(eq(container2URI)); + verify(client).put(eq(binaryURI)); + // no versions should have been created + verify(client, never()).post(versionUri); + } + + @Test + public void testImportVersions() throws Exception { + final URI versionUri = URI.create(containerURI.toString() + "/" + FCR_VERSIONS_PATH); + final URI tombstoneUri = URI.create(container2URI.toString() + "/fcr:tombstone"); + + final HeadBuilder headBuilder = mock(HeadBuilder.class); + final FcrepoResponse headResponse = mock(FcrepoResponse.class); + when(client.head(eq(container2URI))).thenReturn(headBuilder); + when(headBuilder.disableRedirects()).thenReturn(headBuilder); + when(headBuilder.perform()).thenReturn(headResponse); + when(headResponse.getUrl()).thenReturn(container2URI); + when(headResponse.getLinkHeaders(eq("hasTombstone"))).thenReturn(Collections.singletonList(tombstoneUri)); + + ResponseMocker.mockPutResponse(client, containerURI); + ResponseMocker.mockPutResponse(client, container2URI); + ResponseMocker.mockDeleteResponse(client, container2URI); + ResponseMocker.mockDeleteResponse(client, tombstoneUri); + ResponseMocker.mockPutResponse(client, binaryURI); + ResponseMocker.mockPutResponse(client, binaryMDURI); + final PostBuilder versionBuilder = ResponseMocker.mockPostResponse(client, versionUri); + + final Importer importer = new Importer(config, clientBuilder); + importer.run(); + + // Container should have been updated for every version + verify(client, times(3)).put(eq(containerURI)); + verify(client).put(eq(container2URI)); + verify(client, times(2)).put(eq(binaryURI)); + // Verify that the correct number of new versions were created + verify(client, times(2)).post(versionUri); + + verify(versionBuilder).slug("version_1"); + verify(versionBuilder).slug("version_original"); + } + + @Test + public void testVersionsOnWithoutVersionedResources() throws Exception { + config.setBaseDirectory("src/test/resources/sample/container"); + + containerURI = new URI("http://localhost:8080/rest/con1"); + ResponseMocker.mockPutResponse(client, containerURI); + final URI versionUri = URI.create(containerURI.toString() + "/" + FCR_VERSIONS_PATH); + + final Importer importer = new Importer(config, clientBuilder); + importer.run(); + + verify(client).put(eq(containerURI)); + // Ensuring that no versions are created + verify(client, never()).post(versionUri); + } +} diff --git a/src/test/java/org/fcrepo/importexport/importer/ImporterTest.java b/src/test/java/org/fcrepo/importexport/importer/ImporterTest.java index 9b6c3ad2..6ddcfebf 100644 --- a/src/test/java/org/fcrepo/importexport/importer/ImporterTest.java +++ b/src/test/java/org/fcrepo/importexport/importer/ImporterTest.java @@ -50,7 +50,8 @@ import org.fcrepo.client.PutBuilder; import org.fcrepo.importexport.common.AuthenticationRequiredRuntimeException; import org.fcrepo.importexport.common.Config; - +import org.fcrepo.importexport.common.FcrepoConstants; +import org.fcrepo.importexport.test.util.ResponseMocker; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentCaptor; @@ -360,6 +361,31 @@ public void testRepositoryRootFallback() throws FcrepoOperationFailedException { assertEquals(dummy, importer.findRepositoryRoot(dummy)); } + @Test + public void testResourceAndBaseDiffer() throws Exception { + final URI rootURI = URI.create("http://example.org:9999/prod"); + final URI sourceURI = URI.create("http://localhost:8080/rest/dev/asdf"); + final URI resourceURI = URI.create("http://example.org:9999/prod/asdf"); + + mockGet(rootURI, FcrepoConstants.REPOSITORY_ROOT.getURI()); + ResponseMocker.mockHeadResponseError(client, resourceURI, 404); + + ResponseMocker.mockPutResponse(client, resourceURI); + + final Config config = new Config(); + config.setMode("import"); + config.setBaseDirectory("src/test/resources/sample/mapped"); + config.setIncludeBinaries(false); + config.setRdfLanguage("text/turtle"); + config.setResource(resourceURI); + config.setMap(new String[]{sourceURI.toString(), resourceURI.toString()}); + + final Importer importer = new Importer(config, clientBuilder); + importer.run(); + + verify(client).put(resourceURI); + } + private void mockGet(final URI uri, final String type) throws FcrepoOperationFailedException { final GetBuilder getBuilder = mock(GetBuilder.class); final FcrepoResponse getResponse = mock(FcrepoResponse.class); diff --git a/src/test/java/org/fcrepo/importexport/importer/VersionDiffDeletionGeneratorTest.java b/src/test/java/org/fcrepo/importexport/importer/VersionDiffDeletionGeneratorTest.java new file mode 100644 index 00000000..f66a871e --- /dev/null +++ b/src/test/java/org/fcrepo/importexport/importer/VersionDiffDeletionGeneratorTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static org.fcrepo.importexport.common.ModelUtils.mapRdfStream; +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.util.List; + +import org.apache.jena.rdf.model.Model; +import org.fcrepo.importexport.common.Config; +import org.junit.Before; +import org.junit.Test; + +/** + * + * @author bbpennel + * + */ +public class VersionDiffDeletionGeneratorTest { + + private VersionDiffDeletionGenerator generator; + + private Config config; + + @Before + public void init() { + config = new Config(); + config.setRdfLanguage("application/ld+json"); + + generator = new VersionDiffDeletionGenerator(config); + } + + @Test + public void testUnchangedVersion() throws Exception { + final String baseDir = "src/test/resources/sample/versioning/unmodified"; + config.setBaseDirectory(baseDir); + + final Model model = getModel(baseDir + "/rest/v_con1/fcr%3Aversions.jsonld"); + + final List deletions = generator.generateImportDeletions(model); + assertEquals(0, deletions.size()); + } + + @Test + public void testContainerAdded() throws Exception { + // Ensure that containers being added in later versions don't trigger any deletion + final String baseDir = "src/test/resources/sample/versioning/container_added"; + config.setBaseDirectory(baseDir); + + final Model model = getModel(baseDir + "/rest/con1/fcr%3Aversions.jsonld"); + + final List deletions = generator.generateImportDeletions(model); + assertEquals(0, deletions.size()); + } + + @Test + public void testContainerRestoredAdded() throws Exception { + // A container that is removed in a version then added back in should get deleted + final URI child1Uri = URI.create("http://localhost:8080/rest/con1/child1"); + + final String baseDir = "src/test/resources/sample/versioning/container_restored"; + config.setBaseDirectory(baseDir); + + final Model model = getModel(baseDir + "/rest/con1/fcr%3Aversions.jsonld"); + + final List deletions = generator.generateImportDeletions(model); + assertEquals(1, deletions.size()); + assertEquals(child1Uri, deletions.get(0).getUri()); + } + + @Test + public void testContainerRemovedInHeadVersion() throws Exception { + final URI con2Uri = URI.create("http://localhost:8080/fcrepo/rest/con1/con2"); + + final String baseDir = "src/test/resources/sample/versioning/removed_in_head"; + config.setBaseDirectory(baseDir); + + final Model model = getModel(baseDir + "/fcrepo/rest/con1/fcr%3Aversions.jsonld"); + + final List deletions = generator.generateImportDeletions(model); + assertEquals(1, deletions.size()); + assertEquals(con2Uri, deletions.get(0).getUri()); + } + + @Test + public void testBinaryRemoved() throws Exception { + final URI bin1Uri = URI.create("http://localhost:8080/fcrepo/rest/con1/bin1"); + + final String baseDir = "src/test/resources/sample/versioning/binary_removed"; + config.setBaseDirectory(baseDir); + + final Model model = getModel(baseDir + "/fcrepo/rest/con1/fcr%3Aversions.jsonld"); + + final List deletions = generator.generateImportDeletions(model); + assertEquals(1, deletions.size()); + assertEquals(bin1Uri, deletions.get(0).getUri()); + } + + private Model getModel(final String fileUri) throws FileNotFoundException, IOException { + return mapRdfStream(new FileInputStream(new File(fileUri)), config); + } +} diff --git a/src/test/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDFTest.java b/src/test/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDFTest.java new file mode 100644 index 00000000..5b786d61 --- /dev/null +++ b/src/test/java/org/fcrepo/importexport/importer/VersionSubjectMappingStreamRDFTest.java @@ -0,0 +1,172 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.importer; + +import static org.apache.jena.riot.RDFLanguages.contentTypeToLang; +import static org.fcrepo.importexport.common.FcrepoConstants.FEDORA_RESOURCE; +import static org.fcrepo.importexport.common.FcrepoConstants.RDF_TYPE; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.URI; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.ModelFactory; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.riot.RDFFormat; +import org.junit.Test; + +/** + * + * @author bbpennel + * + */ +public class VersionSubjectMappingStreamRDFTest { + + private final String RDF_LANG = "text/turtle"; + private final RDFFormat RDF_FORMAT = RDFFormat.TURTLE_PRETTY; + + private SubjectMappingStreamRDF mapper; + + private URI sourceUri; + private URI destinationUri; + + @Test + public void testRemapBase() throws Exception { + sourceUri = URI.create("http://localhost:9999/rest"); + destinationUri = URI.create("http://example.org:8080/rest"); + + final String rescUri = "http://localhost:9999/rest/con1"; + final String mappedRescUri = "http://example.org:8080/rest/con1"; + + mapper = new VersionSubjectMappingStreamRDF(sourceUri, destinationUri); + + final Model model = makeModelWithResource(rescUri); + + final Model mappedModel = mapModel(model); + final Resource mappedResc = mappedModel.listResourcesWithProperty(RDF_TYPE).next(); + + assertEquals(mappedRescUri, mappedResc.getURI()); + } + + @Test + public void testRemapBaseFromFile() throws Exception { + final String mappedRescUri = "http://localhost:64199/fcrepo/rest/prod2"; + + sourceUri = URI.create("http://localhost:8080/rest/dev/asdf"); + destinationUri = URI.create(mappedRescUri); + + final File mFile = new File("src/test/resources/sample/mapped/rest/dev/asdf.ttl"); + mapper = new SubjectMappingStreamRDF(sourceUri, destinationUri); + + final Model mappedModel; + try (final InputStream in2 = new FileInputStream(mFile)) { + RDFDataMgr.parse(mapper, in2, contentTypeToLang(RDF_LANG)); + } + mappedModel = mapper.getModel(); + + final Resource mappedResc = mappedModel.listResourcesWithProperty(RDF_TYPE).next(); + + assertEquals(mappedRescUri, mappedResc.getURI()); + } + + @Test + public void testNoRemapping() throws Exception { + final String rescUri = "http://localhost:8080/con1"; + + mapper = new VersionSubjectMappingStreamRDF(sourceUri, destinationUri); + + final Model model = makeModelWithResource(rescUri); + + final Model mappedModel = mapModel(model); + final Resource mappedResc = mappedModel.listResourcesWithProperty(RDF_TYPE).next(); + + assertEquals(rescUri, mappedResc.getURI()); + } + + @Test + public void testRemoveVersion() throws Exception { + final String rescUri = "http://localhost:8080/con1"; + final String versionedUri = "http://localhost:8080/con1/fcr:versions/version_1"; + + mapper = new VersionSubjectMappingStreamRDF(sourceUri, destinationUri); + + final Model model = makeModelWithResource(versionedUri); + + final Model mappedModel = mapModel(model); + final Resource mappedResc = mappedModel.listResourcesWithProperty(RDF_TYPE).next(); + + assertEquals(rescUri, mappedResc.getURI()); + } + + @Test + public void testRemoveVersionChild() throws Exception { + final String rescUri = "http://localhost:8080/con1/child"; + final String versionedUri = "http://localhost:8080/con1/fcr:versions/version_1/child"; + + mapper = new VersionSubjectMappingStreamRDF(sourceUri, destinationUri); + + final Model model = makeModelWithResource(versionedUri); + + final Model mappedModel = mapModel(model); + final Resource mappedResc = mappedModel.listResourcesWithProperty(RDF_TYPE).next(); + + assertEquals(rescUri, mappedResc.getURI()); + } + + @Test + public void testRemapBaseAndRemoveVersion() throws Exception { + sourceUri = URI.create("http://localhost:9999/rest"); + destinationUri = URI.create("http://example.org:8080/rest"); + + final String rescUri = "http://localhost:9999/rest/con1/fcr:versions/version_1"; + final String mappedRescUri = "http://example.org:8080/rest/con1"; + + mapper = new VersionSubjectMappingStreamRDF(sourceUri, destinationUri); + + final Model model = makeModelWithResource(rescUri); + + final Model mappedModel = mapModel(model); + final Resource mappedResc = mappedModel.listResourcesWithProperty(RDF_TYPE).next(); + + assertEquals(mappedRescUri, mappedResc.getURI()); + } + + private Model makeModelWithResource(final String uri) { + final Model model = ModelFactory.createDefaultModel(); + final Resource resc = model.getResource(uri); + resc.addProperty(RDF_TYPE, FEDORA_RESOURCE); + + return model; + } + + private Model mapModel(final Model model) throws Exception { + try (final ByteArrayOutputStream bos = new ByteArrayOutputStream()) { + RDFDataMgr.write(bos, model, RDF_FORMAT); + try (final InputStream in2 = new ByteArrayInputStream(bos.toByteArray())) { + RDFDataMgr.parse(mapper, in2, contentTypeToLang(RDF_LANG)); + } + } + return mapper.getModel(); + } +} diff --git a/src/test/java/org/fcrepo/importexport/integration/AbstractResourceIT.java b/src/test/java/org/fcrepo/importexport/integration/AbstractResourceIT.java index 6dc490ef..12f78739 100644 --- a/src/test/java/org/fcrepo/importexport/integration/AbstractResourceIT.java +++ b/src/test/java/org/fcrepo/importexport/integration/AbstractResourceIT.java @@ -32,6 +32,7 @@ import org.fcrepo.client.FcrepoHttpClientBuilder; import org.fcrepo.client.FcrepoOperationFailedException; import org.fcrepo.client.FcrepoResponse; +import org.fcrepo.importexport.common.FcrepoConstants; import org.junit.Before; import org.slf4j.Logger; @@ -42,6 +43,7 @@ import static org.apache.jena.rdf.model.ResourceFactory.createResource; import static org.apache.jena.riot.RDFDataMgr.loadModel; import static org.apache.jena.riot.web.HttpOp.setDefaultHttpClient; +import static org.fcrepo.importexport.common.URITranslationUtil.addRelativePath; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -130,6 +132,13 @@ protected FcrepoResponse createTurtle(final URI uri, final String body) return createBody(uri, body, "text/turtle"); } + protected FcrepoResponse createVersion(final URI uri, final String label) throws FcrepoOperationFailedException { + logger.debug("Create version ---------: {} {}", uri, label); + final URI versionUri = addRelativePath(uri, FcrepoConstants.FCR_VERSIONS_PATH); + + return clientBuilder.build().post(versionUri).slug(label).perform(); + } + protected InputStream insertTitle(final String title) { try { return new ByteArrayInputStream(("INSERT DATA { <> <" + DC_TITLE + "> '" + title + "' . }") diff --git a/src/test/java/org/fcrepo/importexport/integration/ExporterIT.java b/src/test/java/org/fcrepo/importexport/integration/ExporterIT.java index 11a8cd64..363f2931 100644 --- a/src/test/java/org/fcrepo/importexport/integration/ExporterIT.java +++ b/src/test/java/org/fcrepo/importexport/integration/ExporterIT.java @@ -27,8 +27,8 @@ import static org.fcrepo.importexport.common.Config.DEFAULT_RDF_LANG; import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINER; import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; -import static org.fcrepo.importexport.common.FcrepoConstants.HAS_MIME_TYPE; import static org.fcrepo.importexport.common.FcrepoConstants.EXTERNAL_RESOURCE_EXTENSION; +import static org.fcrepo.importexport.common.FcrepoConstants.HAS_MIME_TYPE; import static org.fcrepo.importexport.common.FcrepoConstants.RDF_TYPE; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -213,13 +213,12 @@ public void testExportVersions() throws Exception { final URI res1 = URI.create(baseURI + "/res1"); final URI res2 = URI.create(baseURI + "/res1/res2"); final URI binRes = URI.create(baseURI + "/res1/file"); - final URI res1Versions = URI.create(baseURI + "/res1/fcr:versions"); final String versionLabel = "version1"; create(res1); create(res2); createBody(binRes, "binary", "text/plain"); - createVersion(res1Versions, versionLabel); + createVersion(res1, versionLabel); final Config config = new Config(); config.setMode("export"); @@ -248,10 +247,6 @@ public void testExportVersions() throws Exception { assertTrue(new File(baseDir, "/res1/fcr%3Aversions/version1/file/fcr%3Ametadata" + DEFAULT_RDF_EXT).exists()); } - private void createVersion(final URI uri, final String label) throws FcrepoOperationFailedException { - clientBuilder.build().post(uri).slug(label).perform(); - } - private Config exportWithCustomPredicates(final String[] predicates, final UUID uuid) throws FcrepoOperationFailedException { final String baseURI = serverAddress + uuid; diff --git a/src/test/java/org/fcrepo/importexport/integration/ImporterIT.java b/src/test/java/org/fcrepo/importexport/integration/ImporterIT.java index c28c826b..98cf997f 100644 --- a/src/test/java/org/fcrepo/importexport/integration/ImporterIT.java +++ b/src/test/java/org/fcrepo/importexport/integration/ImporterIT.java @@ -35,16 +35,15 @@ import java.net.URI; import java.util.UUID; +import org.apache.commons.io.IOUtils; import org.apache.jena.graph.Graph; +import org.apache.jena.rdf.model.Model; import org.fcrepo.client.FcrepoClient; import org.fcrepo.client.FcrepoOperationFailedException; import org.fcrepo.client.FcrepoResponse; import org.fcrepo.importexport.common.Config; import org.fcrepo.importexport.exporter.Exporter; import org.fcrepo.importexport.importer.Importer; - -import org.apache.commons.io.IOUtils; -import org.apache.jena.rdf.model.Model; import org.junit.Test; import org.slf4j.Logger; @@ -112,19 +111,19 @@ public void testImport() throws FcrepoOperationFailedException, IOException { binaryText, IOUtils.toString(client.get(binary).perform().getBody(), "UTF-8")); } + @Test public void testCorruptedBinary() throws Exception { - final URI sourceURI = URI.create("http://localhost:8080/fcrepo/rest"); - final URI binaryURI = URI.create("http://localhost:8080/fcrepo/rest/bin1"); + final URI sourceURI = URI.create("http://localhost:8080/rest"); + final URI binaryDestination = URI.create(serverAddress + "rest/bin1"); final String referencePath = TARGET_DIR + "/test-classes/sample/corrupted"; - System.out.println("Importing from " + referencePath); final Config config = new Config(); config.setMode("import"); config.setIncludeBinaries(true); config.setBaseDirectory(referencePath); - config.setRdfLanguage(DEFAULT_RDF_LANG); + config.setRdfLanguage("application/ld+json"); config.setResource(serverAddress); - config.setMap(new String[]{sourceURI.toString(), serverAddress}); + config.setMap(new String[]{sourceURI.toString() + "/", serverAddress}); config.setUsername(USERNAME); config.setPassword(PASSWORD); @@ -133,7 +132,7 @@ public void testCorruptedBinary() throws Exception { importer.run(); // verify that the corrupted binary failed to load - assertFalse(resourceExists(binaryURI)); + assertFalse(resourceExists(binaryDestination)); } @Test diff --git a/src/test/java/org/fcrepo/importexport/integration/RoundtripIT.java b/src/test/java/org/fcrepo/importexport/integration/RoundtripIT.java index c782f368..34d93ea1 100644 --- a/src/test/java/org/fcrepo/importexport/integration/RoundtripIT.java +++ b/src/test/java/org/fcrepo/importexport/integration/RoundtripIT.java @@ -17,6 +17,25 @@ */ package org.fcrepo.importexport.integration; +import static org.apache.http.HttpStatus.SC_CREATED; +import static org.apache.jena.datatypes.xsd.XSDDatatype.XSDdateTime; +import static org.apache.jena.datatypes.xsd.XSDDatatype.XSDlong; +import static org.apache.jena.rdf.model.ResourceFactory.createLangLiteral; +import static org.apache.jena.rdf.model.ResourceFactory.createProperty; +import static org.apache.jena.rdf.model.ResourceFactory.createResource; +import static org.apache.jena.rdf.model.ResourceFactory.createTypedLiteral; +import static org.apache.jena.riot.RDFDataMgr.loadModel; +import static org.fcrepo.importexport.common.Config.DEFAULT_RDF_EXT; +import static org.fcrepo.importexport.common.Config.DEFAULT_RDF_LANG; +import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINER; +import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; +import static org.fcrepo.importexport.common.FcrepoConstants.NON_RDF_SOURCE; +import static org.fcrepo.importexport.common.FcrepoConstants.RDF_TYPE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.slf4j.LoggerFactory.getLogger; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -36,33 +55,15 @@ import org.apache.jena.rdf.model.Resource; import org.apache.jena.util.iterator.ExtendedIterator; import org.fcrepo.client.FcrepoClient; -import org.fcrepo.client.FcrepoResponse; import org.fcrepo.client.FcrepoOperationFailedException; +import org.fcrepo.client.FcrepoResponse; import org.fcrepo.importexport.common.Config; import org.fcrepo.importexport.exporter.Exporter; import org.fcrepo.importexport.importer.Importer; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; -import static org.apache.http.HttpStatus.SC_CREATED; -import static org.apache.jena.datatypes.xsd.XSDDatatype.XSDdateTime; -import static org.apache.jena.datatypes.xsd.XSDDatatype.XSDlong; -import static org.apache.jena.rdf.model.ResourceFactory.createLangLiteral; -import static org.apache.jena.rdf.model.ResourceFactory.createResource; -import static org.apache.jena.rdf.model.ResourceFactory.createProperty; -import static org.apache.jena.rdf.model.ResourceFactory.createTypedLiteral; -import static org.apache.jena.riot.RDFDataMgr.loadModel; -import static org.fcrepo.importexport.common.Config.DEFAULT_RDF_EXT; -import static org.fcrepo.importexport.common.Config.DEFAULT_RDF_LANG; -import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINER; -import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; -import static org.fcrepo.importexport.common.FcrepoConstants.NON_RDF_SOURCE; -import static org.fcrepo.importexport.common.FcrepoConstants.RDF_TYPE; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.slf4j.LoggerFactory.getLogger; - /** * @author escowles * @since 2016-12-07 @@ -223,7 +224,8 @@ public void testRoundtripIndirectContainer() throws Exception { // make sure membership triples were generated by the container client.delete(proxy).perform(); final Model model3 = getAsModel(res2); - assertFalse(model3.contains(parent, createProperty(DCTERMS_HAS_PART), member)); + assertFalse("Membership relation not removed", + model3.contains(parent, createProperty(DCTERMS_HAS_PART), member)); } @Test @@ -439,6 +441,7 @@ public void testRoundtripRdfBinary() throws Exception { // create a binary with an RDF content type final FcrepoResponse response = clientBuilder.build().put(fileURI) .body(new FileInputStream(binaryFile), "text/turtle").filename(null).perform(); + assertEquals(SC_CREATED, response.getStatusCode()); assertEquals(fileURI, response.getLocation()); @@ -449,6 +452,100 @@ public void testRoundtripRdfBinary() throws Exception { assertEquals(binaryContent, getAsString(fileURI)); } + @Test + public void testRoundtripVersions() throws Exception { + final String title = "Updated Version"; + final String baseURI = serverAddress + UUID.randomUUID(); + final URI res1 = URI.create(baseURI + "/res1"); + final URI res2 = URI.create(baseURI + "/res1/res2"); + + create(res1); + createVersion(res1, "v0"); + + create(res2); + createVersion(res1, "v1"); + + final String res1Patch = "insert data { " + + "<> <" + DC_TITLE + "> \"" + title + "\" . }"; + patch(res1, res1Patch); + + roundtrip(res1, true); + + final URI res1V0 = URI.create(baseURI + "/res1/fcr:versions/v0"); + final URI res2V0 = URI.create(baseURI + "/res1/fcr:versions/v0/res2"); + + assertTrue(exists(res1V0)); + assertFalse("res2 is not created until v1", exists(res2V0)); + + final URI res1V1 = URI.create(baseURI + "/res1/fcr:versions/v1"); + final URI res2V1 = URI.create(baseURI + "/res1/fcr:versions/v1/res2"); + + assertTrue(exists(res1V1)); + assertTrue(exists(res2V1)); + + // Check head versions of resources + assertTrue(exists(res1)); + assertTrue(exists(res2)); + + final Model res1Model = getAsModel(res1); + final Model res1V1Model = getAsModel(res1V1); + + assertFalse(res1V1Model.contains(null, createProperty(DC_TITLE), title)); + assertTrue(res1Model.contains(null, createProperty(DC_TITLE), title)); + } + + @Test + public void testRoundtripVersionContainingDelete() throws Exception { + final String baseURI = serverAddress + UUID.randomUUID(); + final URI res1 = URI.create(baseURI + "/res1"); + final URI res2 = URI.create(baseURI + "/res1/res2"); + final URI res3 = URI.create(baseURI + "/res1/res3"); + final URI res4 = URI.create(baseURI + "/res1/res4"); + + // First version contains res1 -> res2, res3 + create(res1); + create(res2); + create(res3); + createVersion(res1, "v0"); + + // Second version contains res1 -> res2, res4 + create(res4); + removeAndReset(res3); + createVersion(res1, "v1"); + + // Head version contains res1 -> rest2 + removeAndReset(res4); + + roundtrip(res1, true); + + final URI res1V0 = URI.create(baseURI + "/res1/fcr:versions/v0"); + final URI res2V0 = URI.create(baseURI + "/res1/fcr:versions/v0/res2"); + final URI res3V0 = URI.create(baseURI + "/res1/fcr:versions/v0/res3"); + final URI res4V0 = URI.create(baseURI + "/res1/fcr:versions/v0/res4"); + + assertTrue(exists(res1V0)); + assertTrue(exists(res2V0)); + assertTrue(exists(res3V0)); + assertFalse("res4 is not created until v1", exists(res4V0)); + + final URI res1V1 = URI.create(baseURI + "/res1/fcr:versions/v1"); + final URI res2V1 = URI.create(baseURI + "/res1/fcr:versions/v1/res2"); + final URI res3V1 = URI.create(baseURI + "/res1/fcr:versions/v1/res3"); + final URI res4V1 = URI.create(baseURI + "/res1/fcr:versions/v1/res4"); + + assertTrue(exists(res1V1)); + assertTrue(exists(res2V1)); + assertFalse("res3 was removed after v0", exists(res3V1)); + assertTrue(exists(res4V1)); + + // Check head versions of resources + assertTrue(exists(res1)); + assertTrue(exists(res2)); + assertFalse("res3 was removed after v0", exists(res3)); + assertFalse("res4 was removed after v1", exists(res4)); + } + + @Ignore("This won't work until 4.7.4 is released to support relaxed server managed triples.") @Test public void testRoundtripNested() throws Exception { final URI containerURI = URI.create(serverAddress + UUID.randomUUID()); @@ -510,6 +607,8 @@ private Config roundtrip(final URI uri, final List relatedResources, config.setRdfLanguage(DEFAULT_RDF_LANG); config.setUsername(USERNAME); config.setPassword(PASSWORD); + config.setIncludeVersions(true); + new Exporter(config, clientBuilder).run(); // delete container and optionally remove tombstone diff --git a/src/test/java/org/fcrepo/importexport/test/util/JsonLdResponse.java b/src/test/java/org/fcrepo/importexport/test/util/JsonLdResponse.java new file mode 100644 index 00000000..45de8f69 --- /dev/null +++ b/src/test/java/org/fcrepo/importexport/test/util/JsonLdResponse.java @@ -0,0 +1,78 @@ +/* + * Licensed to DuraSpace under one or more contributor license agreements. + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * DuraSpace licenses this file to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.fcrepo.importexport.test.util; + +import static org.fcrepo.importexport.common.FcrepoConstants.CONTAINS; +import static org.fcrepo.importexport.common.FcrepoConstants.CREATED_DATE; +import static org.fcrepo.importexport.common.FcrepoConstants.HAS_VERSION; +import static org.fcrepo.importexport.common.FcrepoConstants.HAS_VERSION_LABEL; + +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.jena.rdf.model.Resource; + +/** + * Helpers for constructing json ld responses in tests. + * + * @author bbpennel + * + */ +public abstract class JsonLdResponse { + + public static String createJson(final URI resource, final URI... children) { + return createJson(resource, null, children); + } + + public static String createJson(final URI resource, final Resource type, final URI... children) { + final StringBuilder json = new StringBuilder("{\"@id\":\"" + resource.toString() + "\""); + if (type != null) { + json.append(",\"@type\":[\"" + type.getURI() + "\"]"); + } + if (children != null && children.length > 0) { + json.append(",\"" + CONTAINS.getURI() + "\":[") + .append(Arrays.stream(children) + .map(child -> "{\"@id\":\"" + child.toString() + "\"}") + .collect(Collectors.joining(","))) + .append(']'); + } + json.append('}'); + return json.toString(); + } + + public static String joinJsonArray(final List array) { + return "[" + String.join(",", array) + "]"; + } + + public static List addVersionJson(final List versions, final URI rescUri, final URI versionUri, + final String label, final String timestamp) { + final String versionJson = "{\"@id\":\"" + rescUri.toString() + "\"," + + "\"" + HAS_VERSION.getURI() + "\":[{\"@id\":\"" + versionUri.toString() + "\"}]}," + + "{\"@id\":\"" + versionUri.toString() + "\"," + + "\"" + CREATED_DATE.getURI() + "\":[{" + + "\"@value\":\"" + timestamp + "\"," + + "\"@type\": \"http://www.w3.org/2001/XMLSchema#dateTime\"}]," + + "\"" + HAS_VERSION_LABEL.getURI() + "\":[{" + + "\"@value\":\"" + label + " \"}]" + + "}"; + versions.add(versionJson); + return versions; + } +} diff --git a/src/test/java/org/fcrepo/importexport/test/util/ResponseMocker.java b/src/test/java/org/fcrepo/importexport/test/util/ResponseMocker.java index 2ad6cbb4..222c0302 100644 --- a/src/test/java/org/fcrepo/importexport/test/util/ResponseMocker.java +++ b/src/test/java/org/fcrepo/importexport/test/util/ResponseMocker.java @@ -17,6 +17,8 @@ */ package org.fcrepo.importexport.test.util; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.mock; @@ -27,17 +29,20 @@ import java.net.URI; import java.util.List; +import org.fcrepo.client.DeleteBuilder; import org.fcrepo.client.FcrepoClient; import org.fcrepo.client.FcrepoOperationFailedException; import org.fcrepo.client.FcrepoResponse; import org.fcrepo.client.GetBuilder; import org.fcrepo.client.HeadBuilder; +import org.fcrepo.client.PostBuilder; +import org.fcrepo.client.PutBuilder; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; /** * Test utility for common response mocking behaviors - * + * * @author bbpennel * */ @@ -45,14 +50,15 @@ public abstract class ResponseMocker { /** * Mocks a successful HEAD request response - * + * * @param client client * @param uri uri of destination being mocked * @param typeLinks type links * @param describedbyLinks described by links + * @return the HeadBuilder * @throws FcrepoOperationFailedException client failures */ - public static void mockHeadResponse(final FcrepoClient client, final URI uri, final List typeLinks, + public static HeadBuilder mockHeadResponse(final FcrepoClient client, final URI uri, final List typeLinks, final List describedbyLinks) throws FcrepoOperationFailedException { final HeadBuilder headBuilder = mock(HeadBuilder.class); final FcrepoResponse headResponse = mock(FcrepoResponse.class); @@ -63,19 +69,22 @@ public static void mockHeadResponse(final FcrepoClient client, final URI uri, fi when(headResponse.getLinkHeaders(eq("describedby"))).thenReturn(describedbyLinks); when(headResponse.getStatusCode()).thenReturn(200); when(headResponse.getLinkHeaders(eq("type"))).thenReturn(typeLinks); + + return headBuilder; } /** * Mocks a successful GET request response - * + * * @param client client * @param uri uri of destination being mocked * @param typeLinks type links * @param describedbyLinks described by links * @param body body of response + * @return the GetBuilder * @throws FcrepoOperationFailedException client failures */ - public static void mockGetResponse(final FcrepoClient client, final URI uri, final List typeLinks, + public static GetBuilder mockGetResponse(final FcrepoClient client, final URI uri, final List typeLinks, final List describedbyLinks, final String body) throws FcrepoOperationFailedException { final GetBuilder getBuilder = mock(GetBuilder.class); final FcrepoResponse getResponse = mock(FcrepoResponse.class); @@ -93,17 +102,20 @@ public InputStream answer(final InvocationOnMock invocation) throws Throwable { when(getResponse.getLinkHeaders(eq("describedby"))).thenReturn(describedbyLinks); when(getResponse.getStatusCode()).thenReturn(200); when(getResponse.getLinkHeaders(eq("type"))).thenReturn(typeLinks); + + return getBuilder; } /** * Mocks an unsuccessful GET request response - * + * * @param client client * @param uri uri of destination being mocked * @param statusCode the status code for the response + * @return the GetBuilder * @throws FcrepoOperationFailedException client failures */ - public static void mockGetResponseError(final FcrepoClient client, final URI uri, final int statusCode) + public static GetBuilder mockGetResponseError(final FcrepoClient client, final URI uri, final int statusCode) throws FcrepoOperationFailedException { final GetBuilder getBuilder = mock(GetBuilder.class); final FcrepoResponse getResponse = mock(FcrepoResponse.class); @@ -112,17 +124,20 @@ public static void mockGetResponseError(final FcrepoClient client, final URI uri when(getBuilder.disableRedirects()).thenReturn(getBuilder); when(getBuilder.perform()).thenReturn(getResponse); when(getResponse.getStatusCode()).thenReturn(statusCode); + + return getBuilder; } /** * Mocks an unsuccessful HEAD request response - * + * * @param client client * @param uri uri of destination being mocked * @param statusCode the status code for the response + * @return the HeadBuilder * @throws FcrepoOperationFailedException client failures */ - public static void mockHeadResponseError(final FcrepoClient client, final URI uri, final int statusCode) + public static HeadBuilder mockHeadResponseError(final FcrepoClient client, final URI uri, final int statusCode) throws FcrepoOperationFailedException { final HeadBuilder headBuilder = mock(HeadBuilder.class); final FcrepoResponse response = mock(FcrepoResponse.class); @@ -130,5 +145,74 @@ public static void mockHeadResponseError(final FcrepoClient client, final URI ur when(headBuilder.disableRedirects()).thenReturn(headBuilder); when(headBuilder.perform()).thenReturn(response); when(response.getStatusCode()).thenReturn(statusCode); + + return headBuilder; + } + + /** + * Create a mock PUT response + * + * @param client client + * @param uri uri + * @return the PutBuilder + * @throws FcrepoOperationFailedException thrown by builder + */ + public static PutBuilder mockPutResponse(final FcrepoClient client, final URI uri) + throws FcrepoOperationFailedException { + final PutBuilder putBuilder = mock(PutBuilder.class); + final FcrepoResponse response = mock(FcrepoResponse.class); + when(client.put(eq(uri))).thenReturn(putBuilder); + when(putBuilder.body(isA(InputStream.class), isA(String.class))).thenReturn(putBuilder); + when(putBuilder.digest(isA(String.class))).thenReturn(putBuilder); + when(putBuilder.filename(any())).thenReturn(putBuilder); + when(putBuilder.ifUnmodifiedSince(any())).thenReturn(putBuilder); + when(putBuilder.preferLenient()).thenReturn(putBuilder); + when(putBuilder.perform()).thenReturn(response); + when(response.getStatusCode()).thenReturn(201); + + return putBuilder; + } + + /** + * Create a mock POST response + * + * @param client client + * @param uri uri + * @return the PostBuilder + * @throws FcrepoOperationFailedException thrown by builder + */ + public static PostBuilder mockPostResponse(final FcrepoClient client, final URI uri) + throws FcrepoOperationFailedException { + final PostBuilder postBuilder = mock(PostBuilder.class); + final FcrepoResponse response = mock(FcrepoResponse.class); + when(client.post(eq(uri))).thenReturn(postBuilder); + when(postBuilder.body(isA(InputStream.class), isA(String.class))).thenReturn(postBuilder); + when(postBuilder.digest(isA(String.class))).thenReturn(postBuilder); + when(postBuilder.filename(any())).thenReturn(postBuilder); + when(postBuilder.ifUnmodifiedSince(any())).thenReturn(postBuilder); + when(postBuilder.slug(anyString())).thenReturn(postBuilder); + when(postBuilder.perform()).thenReturn(response); + when(response.getStatusCode()).thenReturn(201); + + return postBuilder; + } + + /** + * Create a mock DELETE response + * + * @param client client + * @param uri uri + * @return response + * @throws FcrepoOperationFailedException thrown by builder + */ + public static DeleteBuilder mockDeleteResponse(final FcrepoClient client, final URI uri) + throws FcrepoOperationFailedException { + final DeleteBuilder deleteBuilder = mock(DeleteBuilder.class); + final FcrepoResponse response = mock(FcrepoResponse.class); + when(client.delete(eq(uri))).thenReturn(deleteBuilder); + when(deleteBuilder.perform()).thenReturn(response); + when(response.getStatusCode()).thenReturn(204); + + return deleteBuilder; } } diff --git a/src/test/resources/sample/recent_parent/rest/con1.jsonld b/src/test/resources/sample/recent_parent/rest/con1.jsonld new file mode 100644 index 00000000..8b513d26 --- /dev/null +++ b/src/test/resources/sample/recent_parent/rest/con1.jsonld @@ -0,0 +1 @@ +[{"@id":"http://localhost:8080/rest/con1","@type":["http://www.w3.org/ns/ldp#RDFSource","http://fedora.info/definitions/v4/repository#Container","http://fedora.info/definitions/v4/repository#Resource","http://www.w3.org/ns/ldp#Container"],"http://fedora.info/definitions/v4/repository#created":[{"@value":"2017-07-27T00:20:02.578Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://fedora.info/definitions/v4/repository#hasParent":[{"@id":"http://localhost:8080/rest/"}],"http://fedora.info/definitions/v4/repository#createdBy":[{"@value":"bypassAdmin"}],"http://purl.org/dc/elements/1.1/title":[{"@value":"Updated 2"}],"http://fedora.info/definitions/v4/repository#lastModifiedBy":[{"@value":"bypassAdmin"}],"http://fedora.info/definitions/v4/repository#hasVersions":[{"@id":"http://localhost:8080/rest/con1/fcr:versions"}],"http://fedora.info/definitions/v4/repository#writable":[{"@value":true}],"http://www.w3.org/ns/ldp#contains":[{"@id":"http://localhost:8080/rest/con1/con2"}],"http://fedora.info/definitions/v4/repository#lastModified":[{"@value":"2017-07-28T01:48:35.519Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}]}] diff --git a/src/test/resources/sample/recent_parent/rest/con1/con2.jsonld b/src/test/resources/sample/recent_parent/rest/con1/con2.jsonld new file mode 100644 index 00000000..be35925b --- /dev/null +++ b/src/test/resources/sample/recent_parent/rest/con1/con2.jsonld @@ -0,0 +1 @@ +[{"@id":"http://localhost:8080/rest/con1/con2","http://fedora.info/definitions/v4/repository#createdBy":[{"@value":"bypassAdmin"}],"@type":["http://fedora.info/definitions/v4/repository#Container","http://www.w3.org/ns/ldp#Container","http://fedora.info/definitions/v4/repository#Resource","http://www.w3.org/ns/ldp#RDFSource"],"http://fedora.info/definitions/v4/repository#created":[{"@value":"2017-07-27T00:20:10.152Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://purl.org/dc/elements/1.1/title":[{"@value":"Child Updated 2"}],"http://fedora.info/definitions/v4/repository#lastModified":[{"@value":"2017-07-28T01:49:20.126Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://fedora.info/definitions/v4/repository#writable":[{"@value":true}],"http://fedora.info/definitions/v4/repository#lastModifiedBy":[{"@value":"bypassAdmin"}],"http://fedora.info/definitions/v4/repository#hasParent":[{"@id":"http://localhost:8080/rest/con1"}]}] diff --git a/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions.jsonld b/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions.jsonld new file mode 100644 index 00000000..127636d5 --- /dev/null +++ b/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions.jsonld @@ -0,0 +1 @@ +[{"@id":"http://localhost:8080/rest/con1","http://fedora.info/definitions/v4/repository#hasVersion":[{"@id":"http://localhost:8080/rest/con1/fcr:versions/v0"}]},{"@id":"http://localhost:8080/rest/con1/fcr:versions/v0","http://fedora.info/definitions/v4/repository#created":[{"@value":"2017-07-27T15:43:41.314Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://fedora.info/definitions/v4/repository#hasVersionLabel":[{"@value":"v0"}]}] diff --git a/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0.jsonld b/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0.jsonld new file mode 100644 index 00000000..eb62f725 --- /dev/null +++ b/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0.jsonld @@ -0,0 +1 @@ +[{"@id":"http://localhost:8080/rest/con1/fcr:versions/v0","http://www.w3.org/ns/ldp#contains":[{"@id":"http://localhost:8080/rest/con1/fcr:versions/v0/con2"}],"http://fedora.info/definitions/v4/repository#created":[{"@value":"2017-07-27T00:20:02.578Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"@type":["http://www.w3.org/ns/ldp#RDFSource","http://fedora.info/definitions/v4/repository#Version","http://www.w3.org/ns/ldp#Container"],"http://fedora.info/definitions/v4/repository#lastModified":[{"@value":"2017-07-27T00:20:47.841Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://fedora.info/definitions/v4/repository#lastModifiedBy":[{"@value":"bypassAdmin"}],"http://fedora.info/definitions/v4/repository#createdBy":[{"@value":"bypassAdmin"}],"http://fedora.info/definitions/v4/repository#writable":[{"@value":true}],"http://purl.org/dc/elements/1.1/title":[{"@value":"Updated"}]}] diff --git a/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0/con2.jsonld b/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0/con2.jsonld new file mode 100644 index 00000000..6590792d --- /dev/null +++ b/src/test/resources/sample/recent_parent/rest/con1/fcr%3Aversions/v0/con2.jsonld @@ -0,0 +1 @@ +[{"@id":"http://localhost:8080/rest/con1/fcr:versions/v0/con2","http://fedora.info/definitions/v4/repository#hasParent":[{"@id":"http://localhost:8080/rest/con1/fcr:versions/v0"}],"http://fedora.info/definitions/v4/repository#writable":[{"@value":true}],"@type":["http://www.w3.org/ns/ldp#Container","http://www.w3.org/ns/ldp#RDFSource","http://fedora.info/definitions/v4/repository#Version"],"http://fedora.info/definitions/v4/repository#lastModified":[{"@value":"2017-07-27T00:20:10.152Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://fedora.info/definitions/v4/repository#created":[{"@value":"2017-07-27T00:20:10.152Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://fedora.info/definitions/v4/repository#createdBy":[{"@value":"bypassAdmin"}],"http://fedora.info/definitions/v4/repository#lastModifiedBy":[{"@value":"bypassAdmin"}]}] diff --git a/src/test/resources/sample/versioned/fcrepo/rest/con1.jsonld b/src/test/resources/sample/versioned/fcrepo/rest/con1.jsonld new file mode 100644 index 00000000..78fa3a6b --- /dev/null +++ b/src/test/resources/sample/versioned/fcrepo/rest/con1.jsonld @@ -0,0 +1 @@ +[{"@id":"http://localhost:8080/fcrepo/rest/con1","http://fedora.info/definitions/v4/repository#writable":[{"@value":true}],"http://fedora.info/definitions/v4/repository#created":[{"@value":"2017-06-09T20:13:29.585Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://fedora.info/definitions/v4/repository#createdBy":[{"@value":"bypassAdmin"}],"@type":["http://fedora.info/definitions/v4/repository#Resource","http://www.w3.org/ns/ldp#RDFSource","http://www.w3.org/ns/ldp#Container","http://fedora.info/definitions/v4/repository#Container"],"http://fedora.info/definitions/v4/repository#hasVersions":[{"@id":"http://localhost:8080/fcrepo/rest/con1/fcr:versions"}],"http://www.w3.org/ns/ldp#contains":[{"@id":"http://localhost:8080/fcrepo/rest/con1/bin1"}],"http://fedora.info/definitions/v4/repository#lastModified":[{"@value":"2017-06-09T20:17:56.246Z","@type":"http://www.w3.org/2001/XMLSchema#dateTime"}],"http://purl.org/dc/elements/1.1/title":[{"@value":"container updated"}],"http://fedora.info/definitions/v4/repository#hasParent":[{"@id":"http://localhost:8080/fcrepo/rest/"}],"http://fedora.info/definitions/v4/repository#lastModifiedBy":[{"@value":"bypassAdmin"}]}] diff --git a/src/test/resources/sample/versioned/fcrepo/rest/con1/bin1.binary b/src/test/resources/sample/versioned/fcrepo/rest/con1/bin1.binary new file mode 100644 index 0000000000000000000000000000000000000000..f1edc52f74f240625608e6f1d3e7a1acc6c28ca9 GIT binary patch literal 32 mcmex=