FCREPO-2459 Import of versions (#109)

* Initial versioned importer
* Added chronological resource iterator
* Moved json ld construction for test to a utility
* Added Subject Mapper for versioned resources
* Excluding versioned resources when generating iterator of import resources if versions are being excluded
* Switched to release version of fcrepo-java-client
* Added constant for Version rdf type
* Refactored import resource factory to shift most of the file discovery over to the iterator
* Changed scope of some components to allow for extension
* Refactored VersionImporter to use a chronological iterator for resources.  Added partial unit test.
* Switched to an event perspective for tracking import of resources and versions.  Incorporated timestamp detection of unmodified resources to prevent unnecessary re-import of resources
* Refactored ImportEvent classes into separate files.  Removed dead code
* Getting more of the tests working
* Addressing ordering issues when parent changes
* Fixed checkstyle issues
* Refactored iterator to default to sorting by created except in the case of versions after the original, which are sorted by last modified
* Got CorruptedBinary IT working
* Replaced Importer with VersionImporter
* Added back in special santizing of membership relations
* Added in verifyBag method for retention purposes
* Pulled in some changes from master to get the last IT working
* Incorporated containerBuilder bag functionality into import of descriptions
* Added round trip IT for versions
* Implemented cleanup of resources that are added in a version but later removed
* Close iterators and correct tests related to the event iterator
* Cleanup tombstones of objects which are deleted in past versions of resources while recreating the versions. Delete unused variables. Update deleteTombstone method to figure out the tombstone uri for the given rescUri if no tombstone uri was given (previously it attempted to get the tombstone uri of the parent, but this was not used). Add integration test to verify import of versions with deleted children.
* Cleaned up unused code related to using digests during binary ingest

Resolves: https://jira.duraspace.org/browse/FCREPO-2459

src/main/java/org/fcrepo/importexport/common/BinaryImportException.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to DuraSpace under one or more contributor license agreements.
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * DuraSpace licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except in
+ * compliance with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.fcrepo.importexport.common;
+
+/**
+ *
+ * @author bbpennel
+ *
+ */
+public class BinaryImportException extends RuntimeException {
+    private static final long serialVersionUID = 1L;
+
+    /**
+     * Constructor
+     *
+     * @param message message
+     */
+    public BinaryImportException(final String message) {
+        super(message);
+    }
+
+    /**
+     * Constructor
+     *
+     * @param message message
+     * @param cause cause
+     */
+    public BinaryImportException(final String message, final Throwable cause) {
+        super(message, cause);
+    }
+
+}

src/main/java/org/fcrepo/importexport/common/FcrepoConstants.java

@@ -44,6 +44,7 @@ public abstract class FcrepoConstants {
     public static final String LDP_NAMESPACE = "http://www.w3.org/ns/ldp#";
     public static final Resource CONTAINER = createResource(LDP_NAMESPACE + "Container");
     public static final Property MEMBERSHIP_RESOURCE = createProperty(LDP_NAMESPACE + "membershipResource");
+    public static final Property HAS_MEMBER_RELATION = createProperty(LDP_NAMESPACE + "hasMemberRelation");
     public static final Property NON_RDF_SOURCE = createProperty(LDP_NAMESPACE + "NonRDFSource");
     public static final Property RDF_SOURCE = createProperty(LDP_NAMESPACE + "RDFSource");
     public static final Property CONTAINS = createProperty(LDP_NAMESPACE + "contains");
@@ -58,7 +59,9 @@ public abstract class FcrepoConstants {
     public static final String REPOSITORY_NAMESPACE = "http://fedora.info/definitions/v4/repository#";
     public static final Resource INBOUND_REFERENCES = createResource(REPOSITORY_NAMESPACE + "InboundReferences");
     public static final Resource PAIRTREE = createResource(REPOSITORY_NAMESPACE + "Pairtree");
+    public static final Resource FEDORA_RESOURCE = createResource(REPOSITORY_NAMESPACE + "Resource");
     public static final Resource REPOSITORY_ROOT = createResource(REPOSITORY_NAMESPACE + "RepositoryRoot");
+    public static final Resource VERSION_RESOURCE = createResource(REPOSITORY_NAMESPACE + "Version");
 
     public static final String BAG_INFO_FIELDNAME = "Bag-Info";
 
@@ -67,7 +70,10 @@ public abstract class FcrepoConstants {
     public static final Property LAST_MODIFIED_DATE = createProperty(REPOSITORY_NAMESPACE + "lastModified");
     public static final Property LAST_MODIFIED_BY = createProperty(REPOSITORY_NAMESPACE + "lastModifiedBy");
 
+    public static final String FCR_METADATA_PATH = "fcr:metadata";
+
     public static final String FCR_VERSIONS_PATH = "fcr:versions";
     public static final Property HAS_VERSION = createProperty(REPOSITORY_NAMESPACE + "hasVersion");
+    public static final Property HAS_VERSIONS = createProperty(REPOSITORY_NAMESPACE + "hasVersions");
     public static final Property HAS_VERSION_LABEL = createProperty(REPOSITORY_NAMESPACE + "hasVersionLabel");
 }

src/main/java/org/fcrepo/importexport/common/ModelUtils.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to DuraSpace under one or more contributor license agreements.
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * DuraSpace licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except in
+ * compliance with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.fcrepo.importexport.common;
+
+import static org.apache.jena.riot.RDFLanguages.contentTypeToLang;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.riot.RDFDataMgr;
+import org.fcrepo.importexport.importer.SubjectMappingStreamRDF;
+import org.fcrepo.importexport.importer.VersionSubjectMappingStreamRDF;
+
+/**
+ * Utilities for manipulating rdf models
+ *
+ * @author bbpennel
+ *
+ */
+public abstract class ModelUtils {
+
+    /**
+     * Parses an input stream of RDF in the configured RDF language into a model, remapping subjects to the configured
+     * destination URL.  This includes removal of version path components if configured.
+     *
+     * @param in RDF input stream
+     * @param config config
+     * @return parsed model
+     * @throws IOException Thrown if the stream cannot be parsed
+     */
+    public static Model mapRdfStreamToNonversionedSubjects(final InputStream in, final Config config)
+            throws IOException {
+        final SubjectMappingStreamRDF mapper;
+        if (config.includeVersions()) {
+            mapper = new VersionSubjectMappingStreamRDF(config.getSource(), config.getDestination());
+        } else {
+            mapper = new SubjectMappingStreamRDF(config.getSource(), config.getDestination());
+        }
+
+        try (final InputStream in2 = in) {
+            RDFDataMgr.parse(mapper, in2, contentTypeToLang(config.getRdfLanguage()));
+        }
+        return mapper.getModel();
+    }
+
+    /**
+     * Parses an input stream of RDF in the configured RDF language into a model, remapping subjects to the configured
+     * destination URL.
+     *
+     * @param in RDF input stream
+     * @param config config
+     * @return parsed model
+     * @throws IOException Thrown if the stream cannot be parsed
+     */
+    public static Model mapRdfStream(final InputStream in, final Config config) throws IOException {
+        final SubjectMappingStreamRDF mapper = new SubjectMappingStreamRDF(config.getSource(),
+                                                                           config.getDestination());
+        try (final InputStream in2 = in) {
+            RDFDataMgr.parse(mapper, in2, contentTypeToLang(config.getRdfLanguage()));
+        }
+        return mapper.getModel();
+    }
+}

src/main/java/org/fcrepo/importexport/common/ResourceGoneRuntimeException.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to DuraSpace under one or more contributor license agreements.
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * DuraSpace licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except in
+ * compliance with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.fcrepo.importexport.common;
+
+import java.net.URI;
+
+import org.fcrepo.client.FcrepoResponse;
+
+/**
+ * Exception thrown when a resource is gone, and checks to see if the resource has a tombstone
+ *
+ * @author bbpennel
+ */
+public class ResourceGoneRuntimeException extends RuntimeException {
+    private static final long serialVersionUID = 1L;
+
+    final URI resourceUri;
+    final URI tombstone;
+
+    /**
+     * Constructor
+     *
+     * @param response Response which triggered the exception
+     */
+    public ResourceGoneRuntimeException(final FcrepoResponse response) {
+        tombstone = response.getLinkHeaders("hasTombstone").get(0);
+        resourceUri = response.getUrl();
+    }
+
+    /**
+     * @return the resourceUri
+     */
+    public URI getResourceUri() {
+        return resourceUri;
+    }
+
+    /**
+     * @return the tombstone
+     */
+    public URI getTombstone() {
+        return tombstone;
+    }
+}

src/main/java/org/fcrepo/importexport/common/TransferProcess.java

@@ -168,7 +168,7 @@ public static File directoryForContainer(final URI uri, final String sourcePath,
      * @return the map
      */
     public static Map<String, String> getSha1FileMap(final File baseDir, final Path manifestFile) {
-        final Map<String, String> sha1FileMap = new HashMap<String, String>();
+        final Map<String, String> sha1FileMap = new HashMap<>();
         try (final Stream<String> stream = Files.lines(manifestFile)) {
             stream.forEach(l -> {
                 final String[] manifestTokens = l.split(BAGIT_CHECKSUM_DELIMITER);

src/main/java/org/fcrepo/importexport/common/URITranslationUtil.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to DuraSpace under one or more contributor license agreements.
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * DuraSpace licenses this file to you under the Apache License,
+ * Version 2.0 (the "License"); you may not use this file except in
+ * compliance with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.fcrepo.importexport.common;
+
+import static org.fcrepo.importexport.common.FcrepoConstants.FCR_VERSIONS_PATH;
+
+import java.io.File;
+import java.net.URI;
+
+/**
+ * Utility for creating or manipulating uris
+ *
+ * @author bbpennel
+ *
+ */
+public abstract class URITranslationUtil {
+
+    /**
+     * Builds the repository URI for the given file
+     *
+     * @param f file to build URI for
+     * @param config config
+     * @return URI for file
+     */
+    public static URI uriForFile(final File f, final Config config) {
+        // get path of file relative to the data directory
+        String relative = config.getBaseDirectory().toPath().relativize(f.toPath()).toString();
+        relative = TransferProcess.decodePath(relative);
+
+        // rebase the path on the destination uri (translating source/destination if needed)
+        if ( config.getSource() != null && config.getDestination() != null ) {
+            relative = baseURI(config.getSource()) + relative;
+            relative = relative.replaceFirst(config.getSource().toString(), config.getDestination().toString());
+        } else {
+            relative = baseURI(config.getResource()) + relative;
+        }
+
+        // for exported RDF, just remove the ".extension" and you have the encoded path
+        if (relative.endsWith(config.getRdfExtension())) {
+            relative = relative.substring(0, relative.length() - config.getRdfExtension().length());
+        }
+        return URI.create(relative);
+    }
+
+    /**
+     * Adds relative path to uri
+     *
+     * @param uri base uri
+     * @param path relative path to add
+     * @return joined uri
+     */
+    public static URI addRelativePath(final URI uri, final String path) {
+        final String base = uri.toString();
+
+        if (base.charAt(base.length() - 1) == '/') {
+            if (path.charAt(0) == '/') {
+                return URI.create(base + path.substring(1, path.length()));
+            }
+            return URI.create(base + path);
+        } else if (path.charAt(0) == '/') {
+            return URI.create(base + path);
+        }
+
+        return URI.create(base + "/" + path);
+    }
+
+    private static String baseURI(final URI uri) {
+        final String base = uri.toString().replaceFirst(uri.getPath() + "$", "");
+        return (base.endsWith("/")) ? base : base + "/";
+    }
+
+    /**
+     * Remaps the given uri to its expected uri within the destination repository
+     *
+     * @param uri resource uri
+     * @param sourceURI source uri
+     * @param destinationURI destination base uri
+     * @return remapped resource uri
+     */
+    public static URI remapResourceUri(final URI uri, final URI sourceURI, final URI destinationURI) {
+        return URI.create(remapResourceUri(uri.toString(), sourceURI == null ? null : sourceURI.toString(),
+                destinationURI == null ? null : destinationURI.toString()));
+    }
+
+    /**
+     * Remaps the given uri to its expected uri within the destination repository
+     *
+     * @param uri resource uri
+     * @param sourceURI source uri
+     * @param destinationURI destination base uri
+     * @return remapped resource uri
+     */
+    public static String remapResourceUri(final String uri, final String sourceURI, final String destinationURI) {
+        String remapped = uri;
+        if (remapped.contains(FCR_VERSIONS_PATH)) {
+            remapped = remapped.replaceFirst("/fcr:versions/[^/]+", "");
+        }
+        if (sourceURI != null && destinationURI != null
+                && uri.startsWith(sourceURI)) {
+            remapped = remapped.replaceFirst(sourceURI, destinationURI);
+        }
+
+        return remapped;
+    }
+}