Merge pull request #511 from marklogic/dev

Merge in changes for 4.8.0

build.gradle

@@ -3,12 +3,12 @@ plugins {
 	id "maven-publish"
 	id "signing"
   id "com.github.jk1.dependency-license-report" version "1.17"
-	id "net.saliman.properties" version "1.5.1"
+	id "net.saliman.properties" version "1.5.2"
 	id "io.snyk.gradle.plugin.snykplugin" version "0.4"
 }
 
 group = "com.marklogic"
-version = "4.7.0"
+version = "4.8.0"
 
 java {
 	sourceCompatibility = 1.8
@@ -24,8 +24,8 @@ repositories {
 }
 
 dependencies {
-	api 'com.marklogic:ml-javaclient-util:4.7.0'
-	api 'org.springframework:spring-web:5.3.31'
+	api 'com.marklogic:ml-javaclient-util:4.8.0'
+	api 'org.springframework:spring-web:5.3.34'
 	api 'com.fasterxml.jackson.core:jackson-databind:2.15.3'
 
 	implementation 'jaxen:jaxen:1.2.0'
@@ -37,7 +37,7 @@ dependencies {
 	implementation 'org.jdom:jdom2:2.0.6.1'
 
 	// Forcing httpclient to use this to address https://snyk.io/vuln/SNYK-JAVA-COMMONSCODEC-561518
-	implementation 'commons-codec:commons-codec:1.15'
+	implementation 'commons-codec:commons-codec:1.16.1'
 
 	// For EqualsBuilder; added in 3.8.1 to support detecting if a mimetype's properties have changed or not
 	implementation "org.apache.commons:commons-lang3:3.14.0"
@@ -57,15 +57,15 @@ dependencies {
 	compileOnly "com.beust:jcommander:1.82"
 	compileOnly "ch.qos.logback:logback-classic:1.3.14"
 
-	testImplementation 'org.junit.jupiter:junit-jupiter:5.10.1'
-	testImplementation 'org.springframework:spring-test:5.3.31'
-	testImplementation 'commons-io:commons-io:2.15.1'
+	testImplementation 'org.junit.jupiter:junit-jupiter:5.10.2'
+	testImplementation 'org.springframework:spring-test:5.3.34'
+	testImplementation 'commons-io:commons-io:2.16.1'
 	testImplementation 'xmlunit:xmlunit:1.6'
 
 	// Forcing Spring to use logback for testing instead of commons-logging
 	testImplementation "ch.qos.logback:logback-classic:1.3.14"
-	testImplementation "org.slf4j:jcl-over-slf4j:1.7.36"
-	testImplementation "org.slf4j:slf4j-api:1.7.36"
+	testImplementation "org.slf4j:jcl-over-slf4j:2.0.13"
+	testImplementation "org.slf4j:slf4j-api:2.0.13"
 }
 
 // This ensures that Gradle includes in the published jar any non-java files under src/main/java

pom.xml

@@ -12,7 +12,7 @@ It is not intended to be used to build this project.
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.marklogic</groupId>
   <artifactId>ml-app-deployer</artifactId>
-  <version>4.6.1</version>
+  <version>4.8.0</version>
   <name>com.marklogic:ml-app-deployer</name>
   <description>Java client for the MarkLogic REST Management API and for deploying applications to MarkLogic</description>
   <url>https://github.com/marklogic/ml-app-deployer</url>

src/main/java/com/marklogic/appdeployer/ConfigDir.java

@@ -262,4 +262,8 @@ public File getProjectDir() {
 	public File getSecureCredentialsDir() {
 		return new File(getSecurityDir(), "secure-credentials");
 	}
+
+	public File getCredentialsDir() {
+		return new File(getSecurityDir(), "credentials");
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -17,7 +17,6 @@
 
 import com.marklogic.appdeployer.util.JavaClientUtil;
 import com.marklogic.client.DatabaseClient;
-import com.marklogic.client.DatabaseClientFactory;
 import com.marklogic.client.ext.SecurityContextType;
 import com.marklogic.mgmt.util.PropertySource;
 import com.marklogic.mgmt.util.PropertySourceFactory;
@@ -250,8 +249,8 @@ public void initialize() {
 		 * different port, in which case you can set this to that port.
 		 */
 		propertyConsumerMap.put("mlAppServicesPort", (config, prop) -> {
-			logger.info("App services port: " + prop);
-			config.setAppServicesPort(Integer.parseInt(prop));
+			logger.info("App services port: {}", prop);
+			config.setAppServicesPort(propertyToInteger("mlAppServicesPort", prop));
 		});
 		/**
 		 * The username and password for a ML user with the rest-admin role that is used for e.g. loading
@@ -376,7 +375,7 @@ public void initialize() {
 		 */
 		propertyConsumerMap.put("mlRestPort", (config, prop) -> {
 			logger.info("App REST port: " + prop);
-			config.setRestPort(Integer.parseInt(prop));
+			config.setRestPort(propertyToInteger("mlRestPort", prop));
 		});
 		/**
 		 * The username and password for a ML user with the rest-admin role. This user is used for operations against the
@@ -559,7 +558,7 @@ public void initialize() {
 		 */
 		propertyConsumerMap.put("mlTestRestPort", (config, prop) -> {
 			logger.info("Test REST port: " + prop);
-			config.setTestRestPort(Integer.parseInt(prop));
+			config.setTestRestPort(propertyToInteger("mlTestRestPort", prop));
 		});
 
 		propertyConsumerMap.put("mlTestRestServerName", (config, prop) -> {
@@ -605,7 +604,7 @@ public void initialize() {
 
 		propertyConsumerMap.put("mlContentForestsPerHost", (config, prop) -> {
 			logger.info("Content forests per host: " + prop);
-			config.setContentForestsPerHost(Integer.parseInt(prop));
+			config.setContentForestsPerHost(propertyToInteger("mlContentForestsPerHost", prop));
 		});
 
 		propertyConsumerMap.put("mlCreateForests", (config, prop) -> {
@@ -620,7 +619,7 @@ public void initialize() {
 			logger.info("Forests per host: " + prop);
 			String[] tokens = prop.split(",");
 			for (int i = 0; i < tokens.length; i += 2) {
-				config.getForestCounts().put(tokens[i], Integer.parseInt(tokens[i + 1]));
+				config.getForestCounts().put(tokens[i], propertyToInteger("mlForestsPerHost", tokens[i + 1]));
 			}
 		});
 
@@ -633,7 +632,7 @@ public void initialize() {
 			String[] tokens = prop.split(",");
 			Map<String, Integer> map = new HashMap<>();
 			for (int i = 0; i < tokens.length; i += 2) {
-				map.put(tokens[i], Integer.parseInt(tokens[i + 1]));
+				map.put(tokens[i], propertyToInteger("mlDatabaseNamesAndReplicaCounts", tokens[i + 1]));
 			}
 			config.setDatabaseNamesAndReplicaCounts(map);
 		});
@@ -884,12 +883,12 @@ public void initialize() {
 
 		propertyConsumerMap.put("mlModulesLoaderThreadCount", (config, prop) -> {
 			logger.info("Modules loader thread count: " + prop);
-			config.setModulesLoaderThreadCount(Integer.parseInt(prop));
+			config.setModulesLoaderThreadCount(propertyToInteger("mlModulesLoaderThreadCount", prop));
 		});
 
 		propertyConsumerMap.put("mlModulesLoaderBatchSize", (config, prop) -> {
 			logger.info("Modules loader batch size: " + prop);
-			config.setModulesLoaderBatchSize(Integer.parseInt(prop));
+			config.setModulesLoaderBatchSize(propertyToInteger("mlModulesLoaderBatchSize", prop));
 		});
 
 		propertyConsumerMap.put("mlCascadeCollections", (config, prop) -> {
@@ -1000,7 +999,7 @@ public void initialize() {
 	protected void registerDataLoadingProperties() {
 		propertyConsumerMap.put("mlDataBatchSize", (config, prop) -> {
 			logger.info("Batch size for loading data: " + prop);
-			config.getDataConfig().setBatchSize(Integer.parseInt(prop));
+			config.getDataConfig().setBatchSize(propertyToInteger("mlDataBatchSize", prop));
 		});
 
 		propertyConsumerMap.put("mlDataCollections", (config, prop) -> {

src/main/java/com/marklogic/appdeployer/command/CommandMapBuilder.java

@@ -161,6 +161,7 @@ private void addCommandsThatDoNotWriteToDatabases(Map<String, List<Command>> map
 		securityCommands.add(new InsertCertificateHostsTemplateCommand());
 		securityCommands.add(new DeployExternalSecurityCommand());
 		securityCommands.add(new DeploySecureCredentialsCommand());
+		securityCommands.add(new DeployCredentialsCommand());
 		securityCommands.add(new DeployPrivilegesCommand());
 		securityCommands.add(new DeployPrivilegeRolesCommand());
 		securityCommands.add(new DeployProtectedCollectionsCommand());

src/main/java/com/marklogic/appdeployer/command/SortOrderConstants.java

@@ -33,6 +33,7 @@ public abstract class SortOrderConstants {
 
 	public static Integer DEPLOY_EXTERNAL_SECURITY = 35;
 	public static Integer DEPLOY_SECURE_CREDENTIALS = 36;
+	public static Integer DEPLOY_CREDENTIALS = 37;
 	public static Integer DEPLOY_PROTECTED_COLLECTIONS = 40;
 	public static Integer DEPLOY_MIMETYPES = 45;
 

src/main/java/com/marklogic/appdeployer/command/security/DeployCredentialsCommand.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2023 MarkLogic Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.appdeployer.command.security;
+
+import com.marklogic.appdeployer.command.AbstractResourceCommand;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.appdeployer.command.SortOrderConstants;
+import com.marklogic.appdeployer.command.UndoableCommand;
+import com.marklogic.mgmt.resource.ResourceManager;
+import com.marklogic.mgmt.resource.security.CredentialsManager;
+
+import java.io.File;
+
+public class DeployCredentialsCommand extends AbstractResourceCommand implements UndoableCommand {
+
+	public DeployCredentialsCommand() {
+		setExecuteSortOrder(SortOrderConstants.DEPLOY_CREDENTIALS);
+		setUndoSortOrder(SortOrderConstants.DEPLOY_CREDENTIALS);
+	}
+
+	@Override
+	protected File[] getResourceDirs(CommandContext context) {
+		return findResourceDirs(context, configDir -> configDir.getCredentialsDir());
+	}
+
+	@Override
+	protected ResourceManager getResourceManager(CommandContext context) {
+		return new CredentialsManager(context.getManageClient());
+	}
+}

src/main/java/com/marklogic/mgmt/DefaultManageConfigFactory.java

@@ -56,7 +56,7 @@ public void initialize() {
 
 	    propertyConsumerMap.put("mlManagePort", (config, prop) -> {
 		    logger.info("Manage port: " + prop);
-		    config.setPort(Integer.parseInt(prop));
+		    config.setPort(propertyToInteger("mlManagePort", prop));
 	    });
 
 		propertyConsumerMap.put("mlManageAuthentication", (config, prop) -> {

src/main/java/com/marklogic/mgmt/ManageClient.java

@@ -22,6 +22,7 @@
 import com.marklogic.rest.util.RestConfig;
 import com.marklogic.rest.util.RestTemplateUtil;
 import org.jdom2.Namespace;
+import org.springframework.core.io.Resource;
 import org.springframework.http.*;
 import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.util.MultiValueMap;
@@ -204,16 +205,28 @@ public String getJsonAsSecurityUser(String path) {
 			.getBody();
 	}
 
-	public void delete(String path) {
+	public void delete(String path, String... headerNamesAndValues) {
         logRequest(path, "", "DELETE");
-        getRestTemplate().delete(buildUri(path));
+		delete(getRestTemplate(), path, headerNamesAndValues);
     }
 
-    public void deleteAsSecurityUser(String path) {
+    public void deleteAsSecurityUser(String path, String... headerNamesAndValues) {
 	    logSecurityUserRequest(path, "", "DELETE");
-	    getSecurityUserRestTemplate().delete(buildUri(path));
+		delete(getSecurityUserRestTemplate(), path, headerNamesAndValues);
     }
 
+	private void delete(RestTemplate restTemplate, String path, String... headerNamesAndValues) {
+		URI uri = buildUri(path);
+		HttpHeaders headers = new HttpHeaders();
+		if (headerNamesAndValues != null) {
+			for (int i = 0; i < headerNamesAndValues.length; i += 2) {
+				headers.add(headerNamesAndValues[i], headerNamesAndValues[i + 1]);
+			}
+		}
+		HttpEntity<Resource> entity = new HttpEntity<>(null, headers);
+		restTemplate.exchange(uri, HttpMethod.DELETE, entity, String.class);
+	}
+
 	/**
 	 * Per #187 and version 3.1.0, when an HttpEntity is constructed with a JSON payload, this method will check to see
 	 * if it should "clean" the JSON via the Jackson library, which is primarily intended for removing comments from

src/main/java/com/marklogic/mgmt/admin/AdminManager.java

@@ -19,6 +19,7 @@
 import com.marklogic.rest.util.Fragment;
 import com.marklogic.rest.util.RestConfig;
 import com.marklogic.rest.util.RestTemplateUtil;
+import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.springframework.core.io.ByteArrayResource;
 import org.springframework.core.io.Resource;
 import org.springframework.http.*;
@@ -80,8 +81,9 @@ public boolean execute() {
                 HttpHeaders headers = new HttpHeaders();
                 headers.setContentType(MediaType.APPLICATION_JSON);
                 HttpEntity<String> entity = new HttpEntity<>(payload, headers);
+				ImmutablePair<Boolean, Boolean> credentialsStatus = checkCredentialsAndReplaceNulls(adminConfig);
                 try {
-                    ResponseEntity<String> response = getRestTemplate().exchange(uri, HttpMethod.POST, entity, String.class);
+					ResponseEntity<String> response = getRestTemplate().exchange(uri, HttpMethod.POST, entity, String.class);
                     logger.info("Initialization response: " + response);
                     // According to http://docs.marklogic.com/REST/POST/admin/v1/init, a 202 is sent back in the event a
                     // restart is needed. A 400 or 401 will be thrown as an error by RestTemplate.
@@ -98,12 +100,14 @@ public boolean execute() {
                         logger.error("Caught error, response body: " + body);
                         throw hcee;
                     }
-                }
+                } finally {
+					restoreCredentials(adminConfig, credentialsStatus);
+				}
             }
         });
     }
 
-    public void installAdmin() {
+	public void installAdmin() {
         installAdmin(null, null);
     }
 
@@ -130,8 +134,9 @@ public boolean execute() {
                 HttpHeaders headers = new HttpHeaders();
                 headers.setContentType(MediaType.APPLICATION_JSON);
                 HttpEntity<String> entity = new HttpEntity<>(payload, headers);
+				ImmutablePair<Boolean, Boolean> credentialsStatus = checkCredentialsAndReplaceNulls(adminConfig);
                 try {
-                    ResponseEntity<String> response = getRestTemplate().exchange(uri, HttpMethod.POST, entity, String.class);
+					ResponseEntity<String> response = getRestTemplate().exchange(uri, HttpMethod.POST, entity, String.class);
                     logger.info("Admin installation response: " + response);
                     // According to http://docs.marklogic.com/REST/POST/admin/v1/init, a 202 is sent back in the event a
                     // restart is needed. A 400 or 401 will be thrown as an error by RestTemplate.
@@ -143,6 +148,8 @@ public boolean execute() {
                         return false;
                     }
                     throw hcee;
+				} finally {
+					restoreCredentials(adminConfig, credentialsStatus);
                 }
             }
         });
@@ -323,4 +330,31 @@ public RestTemplate getRestTemplate() {
 		}
 		return this.restTemplate;
 	}
+
+	private ImmutablePair<Boolean, Boolean> checkCredentialsAndReplaceNulls(AdminConfig adminConfig) {
+		boolean setNullUsername = false;
+		boolean setNullPassword = false;
+		if (adminConfig.getUsername() == null) {
+			adminConfig.setUsername("");
+			setNullUsername = true;
+			this.restTemplate = null;
+		}
+		if (adminConfig.getPassword() == null) {
+			adminConfig.setPassword("");
+			setNullPassword = true;
+			this.restTemplate = null;
+		}
+		return new ImmutablePair<>(setNullUsername, setNullPassword);
+	}
+
+	private void restoreCredentials(AdminConfig adminConfig, ImmutablePair<Boolean, Boolean> credentialsStatus) {
+		if (credentialsStatus.getLeft()) {
+			adminConfig.setUsername(null);
+			this.restTemplate = null;
+		}
+		if (credentialsStatus.getRight()) {
+			adminConfig.setPassword(null);
+			this.restTemplate = null;
+		}
+	}
 }

src/main/java/com/marklogic/mgmt/admin/DefaultAdminConfigFactory.java

@@ -56,7 +56,7 @@ public void initialize() {
 
 	    propertyConsumerMap.put("mlAdminPort", (config, prop) -> {
 		    logger.info("Admin interface port: " + prop);
-		    config.setPort(Integer.parseInt(prop));
+		    config.setPort(propertyToInteger("mlAdminPort", prop));
 	    });
 
 		propertyConsumerMap.put("mlAdminAuthentication", (config, prop) -> {

src/main/java/com/marklogic/mgmt/api/database/DatabaseSorter.java

@@ -43,6 +43,13 @@ public String[] sortDatabasesAndReturnNames(List<Database> databases) {
 			}
 		}
 
-		return sorter.sort();
+		try {
+			return sorter.sort();
+		} catch (IllegalStateException ex) {
+			throw new IllegalArgumentException("Unable to deploy databases due to circular dependencies " +
+				"between two or more databases; please remove these circular dependencies in order to deploy" +
+				" your databases. An example of a circular dependency is database A depending on database B as its " +
+				"triggers databases, while database B depends on database A as its schemas database.");
+		}
 	}
 }