#53 Can now specify a collections.properties file to configure collec…

…tions for a ruleset

src/main/java/com/marklogic/client/file/CollectionsDocumentFileProcessor.java

@@ -0,0 +1,83 @@
+package com.marklogic.client.file;
+
+import com.marklogic.client.helper.LoggingObject;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Looks for a special file in each directory - defaults to collections.properties - that contains properties where the
+ * key is the name of a file in the directory, and the value is a comma-delimited list of collections to load the file
+ * into (which means you can't use a comma in any collection name). This will by default NOT load the configured
+ * properties file, as that's expected to just be configuration data and not something that's intended to be loaded
+ * into MarkLogic.
+ */
+public class CollectionsDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
+
+	private String collectionsFilename = "collections.properties";
+
+	// Used to avoid checking for and loading the properties for every file in a directory
+	private Map<File, Properties> propertiesCache = new HashMap<>();
+
+	/**
+	 * @param documentFile
+	 * @return
+	 */
+	@Override
+	public DocumentFile processDocumentFile(DocumentFile documentFile) {
+		File file = documentFile.getFile();
+		String name = file.getName();
+		if (collectionsFilename.equals(name)) {
+			return null;
+		}
+
+		File collectionsFile = new File(file.getParentFile(), collectionsFilename);
+		if (collectionsFile.exists()) {
+			try {
+				Properties props = loadProperties(collectionsFile);
+				if (props.containsKey(name)) {
+					String value = props.getProperty(name);
+					documentFile.getDocumentMetadata().withCollections(value.split(","));
+				}
+			} catch (IOException e) {
+				logger.warn("Unable to load properties from collections file: " + collectionsFile.getAbsolutePath(), e);
+			}
+		}
+
+		return documentFile;
+	}
+
+	protected Properties loadProperties(File collectionsFile) throws IOException {
+		Properties props = null;
+		if (propertiesCache.containsKey(collectionsFile)) {
+			props = propertiesCache.get(collectionsFile);
+		}
+		if (props != null) {
+			return props;
+		}
+		props = new Properties();
+		FileReader reader = null;
+		try {
+			reader = new FileReader(collectionsFile);
+			props.load(reader);
+			propertiesCache.put(collectionsFile, props);
+			return props;
+		} finally {
+			if (reader != null) {
+				reader.close();
+			}
+		}
+	}
+
+	public Map<File, Properties> getPropertiesCache() {
+		return propertiesCache;
+	}
+
+	public void setCollectionsFilename(String collectionsFilename) {
+		this.collectionsFilename = collectionsFilename;
+	}
+}

src/main/java/com/marklogic/client/file/DefaultFileLoader.java

@@ -30,26 +30,32 @@ public DefaultFileLoader(BatchWriter batchWriter, DocumentFileFinder documentFil
 
 	protected void initializeDocumentFileProcessors() {
 		documentFileProcessors = new ArrayList<>();
+		documentFileProcessors.add(new CollectionsDocumentFileProcessor());
 		documentFileProcessors.add(new FormatDocumentFileProcessor());
 	}
 
 	@Override
 	public List<DocumentFile> loadFiles(String... paths) {
 		List<DocumentFile> documentFiles = documentFileFinder.findDocumentFiles(paths);
-		processDocumentFiles(documentFiles);
+		documentFiles = processDocumentFiles(documentFiles);
 		batchWriter.write(documentFiles);
 		return documentFiles;
 	}
 
 	protected List<DocumentFile> processDocumentFiles(List<DocumentFile> documentFiles) {
+		List<DocumentFile> newFiles = new ArrayList<>();
 		for (DocumentFile file : documentFiles) {
 			for (DocumentFileProcessor processor : documentFileProcessors) {
-				if (processor.supportsDocumentFile(file)) {
-					processor.processDocumentFile(file);
+				file = processor.processDocumentFile(file);
+				if (file == null) {
+					break;
 				}
 			}
+			if (file != null) {
+				newFiles.add(file);
+			}
 		}
-		return documentFiles;
+		return newFiles;
 	}
 	public DocumentFileProcessor getDocumentFileProcessor(String classShortName) {
 		for (DocumentFileProcessor processor : documentFileProcessors) {

src/main/java/com/marklogic/client/file/DocumentFileFinder.java

@@ -2,6 +2,10 @@
 
 import java.util.List;
 
+/**
+ * Strategy interface for determining which files to load into MarkLogic, with those files being captured as a List of
+ * DocumentFile objects.
+ */
 public interface DocumentFileFinder {
 
 	List<DocumentFile> findDocumentFiles(String... paths);

src/main/java/com/marklogic/client/file/DocumentFileProcessor.java

@@ -1,12 +1,14 @@
 package com.marklogic.client.file;
 
 /**
- * Callback-style interface for processing - i.e. do whatever you want with - a DocumentFile instance before it's
- * written to MarkLogic.
+ * Interface for processing - i.e. do whatever you want with - a DocumentFile instance before it's written to MarkLogic.
  */
 public interface DocumentFileProcessor {
 
-	boolean supportsDocumentFile(DocumentFile documentFile);
+	/**
+	 * @param documentFile
+	 * @return the same or a new DocumentFile instance
+	 */
+	DocumentFile processDocumentFile(DocumentFile documentFile);
 
-	void processDocumentFile(DocumentFile documentFile);
 }

src/main/java/com/marklogic/client/file/FilterDocumentFileProcessor.java

@@ -0,0 +1,20 @@
+package com.marklogic.client.file;
+
+import java.io.FileFilter;
+
+/**
+ * Simple filter implementation for returning null if the DocumentFile doesn't match the given FileFilter.
+ */
+public class FilterDocumentFileProcessor implements DocumentFileProcessor {
+
+	private FileFilter fileFilter;
+
+	public FilterDocumentFileProcessor(FileFilter fileFilter) {
+		this.fileFilter = fileFilter;
+	}
+
+	@Override
+	public DocumentFile processDocumentFile(DocumentFile documentFile) {
+		return fileFilter.accept(documentFile.getFile()) ? documentFile : null;
+	}
+}

src/main/java/com/marklogic/client/file/FormatDocumentFileProcessor.java

@@ -21,16 +21,12 @@ public FormatDocumentFileProcessor(FormatGetter formatGetter) {
 	}
 
 	@Override
-	public boolean supportsDocumentFile(DocumentFile documentFile) {
-		return true;
-	}
-
-	@Override
-	public void processDocumentFile(DocumentFile documentFile) {
+	public DocumentFile processDocumentFile(DocumentFile documentFile) {
 		Format format = formatGetter.getFormat(documentFile.getFile());
 		if (format != null) {
 			documentFile.setFormat(format);
 		}
+		return documentFile;
 	}
 
 	public FormatGetter getFormatGetter() {

src/main/java/com/marklogic/client/schemasloader/impl/TdeDocumentFileProcessor.java

@@ -7,20 +7,18 @@
 public class TdeDocumentFileProcessor implements DocumentFileProcessor {
 
 	@Override
-	public boolean supportsDocumentFile(DocumentFile documentFile) {
+	public DocumentFile processDocumentFile(DocumentFile documentFile) {
 		String extension = documentFile.getFileExtension();
-		return "tdej".equals(extension) || "tdex".equals(extension);
-	}
-
-	@Override
-	public void processDocumentFile(DocumentFile documentFile) {
-		documentFile.getDocumentMetadata().withCollections("http://marklogic.com/xdmp/tde");
 
-		String extension = documentFile.getFileExtension();
-		if ("tdej".equals(extension)) {
-			documentFile.setFormat(Format.JSON);
-		} else if ("tdex".equals(extension)) {
-			documentFile.setFormat(Format.XML);
+		if ("tdej".equals(extension) || "tdex".equals(extension)) {
+			documentFile.getDocumentMetadata().withCollections("http://marklogic.com/xdmp/tde");
+			if ("tdej".equals(extension)) {
+				documentFile.setFormat(Format.JSON);
+			} else if ("tdex".equals(extension)) {
+				documentFile.setFormat(Format.XML);
+			}
 		}
+
+		return documentFile;
 	}
 }

src/test/java/com/marklogic/client/schemasloader/impl/LoadRulesetsTest.java

@@ -0,0 +1,44 @@
+package com.marklogic.client.schemasloader.impl;
+
+import com.marklogic.client.AbstractIntegrationTest;
+import com.marklogic.client.file.DocumentFile;
+import com.marklogic.client.helper.ClientHelper;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * Created by rrudin on 2/21/2017.
+ */
+public class LoadRulesetsTest extends AbstractIntegrationTest {
+
+	/**
+	 * Wipes out documents matching the ones we intend to load - it's assumed you're not using the Schemas database for
+	 * anything besides ad hoc testing like this.
+	 */
+	@Before
+	public void setup() {
+		client = newClient("Schemas");
+		client.newServerEval().xquery("cts:uri-match('/ruleset*.*') ! xdmp:document-delete(.)").eval();
+	}
+
+	@Test
+	public void test() {
+		DefaultSchemasLoader loader = new DefaultSchemasLoader(client);
+		List<DocumentFile> files = loader.loadSchemas("src/test/resources/rulesets/collection-test");
+		assertEquals(2, files.size());
+
+		ClientHelper helper = new ClientHelper(client);
+
+		List<String> collections = helper.getCollections("/ruleset1.xml");
+		assertEquals(1, collections.size());
+		assertTrue(collections.contains("ruleset-abc"));
+
+		collections = helper.getCollections("/ruleset2.json");
+		assertEquals(2, collections.size());
+		assertTrue(collections.contains("ruleset-abc"));
+		assertTrue(collections.contains("ruleset-xyz"));
+
+	}
+}

src/test/resources/rulesets/collection-test/collections.properties

@@ -0,0 +1,2 @@
+ruleset1.xml=ruleset-abc
+ruleset2.json=ruleset-abc,ruleset-xyz

src/test/resources/rulesets/collection-test/ruleset1.xml

@@ -0,0 +1,9 @@
+<rdt:rule xml:lang="zxx" xmlns:rdt="http://marklogic.com/xdmp/redaction">
+	<rdt:path>//name</rdt:path>
+	<rdt:method>
+		<rdt:function>mask-deterministic</rdt:function>
+	</rdt:method>
+	<rdt:options>
+		<length>10</length>
+	</rdt:options>
+</rdt:rule>

src/test/resources/rulesets/collection-test/ruleset2.json

@@ -0,0 +1,10 @@
+{
+	"description": "hide SSNs",
+	"path": "//ssn",
+	"method": {
+		"function": "redact-us-ssn"
+	},
+	"options": {
+		"pattern": "partial"
+	}
+}