From c38f2878d7820b9f46f9b821f63e355f0581b0a3 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 13 Jun 2017 15:59:58 -0400 Subject: [PATCH] #165 Uses new approach for loading modules From ml-javaclient-util --- build.gradle | 5 + gradle.properties | 4 +- .../com/marklogic/appdeployer/AppConfig.java | 133 ++++------- .../appdeployer/command/AbstractCommand.java | 8 +- ....java => DefaultPayloadTokenReplacer.java} | 2 +- ...eplacer.java => PayloadTokenReplacer.java} | 4 +- .../UpdateRestApiServersCommand.java | 4 +- .../clusters/ModifyLocalClusterCommand.java | 2 +- .../DeployContentDatabasesCommand.java | 6 +- .../databases/DeployDatabaseCommand.java | 10 +- .../command/forests/DeployForestsCommand.java | 2 +- .../modules/DefaultModulesLoaderFactory.java | 98 +++++++++ .../command/modules/LoadModulesCommand.java | 207 +++++++++--------- .../command/modules/ModulesLoaderFactory.java | 13 ++ .../restapis/DeployRestApiServersCommand.java | 6 +- .../appdeployer/util/ModulesWatcher.java | 92 ++++---- .../appdeployer/AbstractAppDeployerTest.java | 5 +- .../es/GenerateModelArtifactsTest.java | 2 +- .../command/modules/LoadModulesTest.java | 39 ++-- .../CreateRestApiAsNonAdminUserTest.java | 6 +- .../appdeployer/export/ExportServerTest.java | 2 +- .../security/roles/sample-app-role.json | 8 +- 22 files changed, 378 insertions(+), 280 deletions(-) rename src/main/java/com/marklogic/appdeployer/command/{DefaultTokenReplacer.java => DefaultPayloadTokenReplacer.java} (95%) rename src/main/java/com/marklogic/appdeployer/command/{TokenReplacer.java => PayloadTokenReplacer.java} (75%) create mode 100644 src/main/java/com/marklogic/appdeployer/command/modules/DefaultModulesLoaderFactory.java create mode 100644 src/main/java/com/marklogic/appdeployer/command/modules/ModulesLoaderFactory.java diff --git a/build.gradle b/build.gradle index 366a7bd3c..12d37074c 100644 --- a/build.gradle +++ b/build.gradle @@ -26,6 +26,11 @@ dependencies { } testCompile('commons-io:commons-io:2.5') + + // Forcing Spring to use logback instead of commons-logging + runtime "ch.qos.logback:logback-classic:1.1.8" + runtime group: "org.slf4j", name: "jcl-over-slf4j", version: "1.7.22" + runtime group: "org.slf4j", name: "slf4j-api", version: "1.7.22" } // This ensures that Gradle includes in the published jar any non-java files under src/main/java diff --git a/gradle.properties b/gradle.properties index 99c6bf3e5..414c6414a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ group=com.marklogic javadocsDir=../gh-pages-marklogic-java/javadocs -version=2.7.0 -mlJavaclientUtilVersion=2.14.0 +version=DEV +mlJavaclientUtilVersion=45 diff --git a/src/main/java/com/marklogic/appdeployer/AppConfig.java b/src/main/java/com/marklogic/appdeployer/AppConfig.java index dd6aeabc2..15e07fd92 100644 --- a/src/main/java/com/marklogic/appdeployer/AppConfig.java +++ b/src/main/java/com/marklogic/appdeployer/AppConfig.java @@ -4,20 +4,16 @@ import com.marklogic.client.DatabaseClientFactory; import com.marklogic.client.DatabaseClientFactory.Authentication; import com.marklogic.client.DatabaseClientFactory.SSLHostnameVerifier; -import com.marklogic.client.modulesloader.impl.StaticChecker; -import com.marklogic.client.modulesloader.impl.XccAssetLoader; -import com.marklogic.client.modulesloader.impl.XccStaticChecker; +import com.marklogic.client.ext.tokenreplacer.PropertiesSource; +import com.marklogic.client.modulesloader.impl.PropertiesModuleManager; import com.marklogic.client.modulesloader.ssl.SimpleX509TrustManager; -import com.marklogic.client.modulesloader.tokenreplacer.DefaultModuleTokenReplacer; -import com.marklogic.client.modulesloader.tokenreplacer.ModuleTokenReplacer; -import com.marklogic.client.modulesloader.tokenreplacer.PropertiesSource; -import com.marklogic.client.modulesloader.tokenreplacer.RoxyModuleTokenReplacer; -import com.marklogic.client.modulesloader.xcc.DefaultDocumentFormatGetter; -import com.marklogic.xcc.template.XccTemplate; import javax.net.ssl.SSLContext; import java.io.FileFilter; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * Encapsulates common configuration properties for an application deployed to MarkLogic. These properties include not @@ -68,6 +64,10 @@ public class AppConfig { private SSLHostnameVerifier restSslHostnameVerifier; private Authentication restAuthentication = Authentication.DIGEST; + private SSLContext appServicesSslContext; + private SSLHostnameVerifier appServicesSslHostnameVerifier; + private Authentication appServicesAuthentication = Authentication.DIGEST; + private Integer restPort = DEFAULT_PORT; private Integer testRestPort; private Integer appServicesPort = 8000; @@ -87,12 +87,12 @@ public class AppConfig { private boolean staticCheckAssets = false; private boolean staticCheckLibraryAssets = false; private boolean bulkLoadAssets = true; - private String moduleTimestampsPath; + private String moduleTimestampsPath = PropertiesModuleManager.DEFAULT_FILE_PATH; private String schemasPath; private ConfigDir configDir; - // Passed into the TokenReplacer that subclasses of AbstractCommand use + // Passed into the PayloadTokenReplacer that subclasses of AbstractCommand use private Map customTokens = new HashMap<>(); // Allows for creating a triggers database without a config file for one @@ -183,88 +183,21 @@ public DatabaseClient newTestDatabaseClient() { getRestAdminPassword(), getRestAuthentication(), getRestSslContext(), getRestSslHostnameVerifier()); } + public DatabaseClient newModulesDatabaseClient() { + return DatabaseClientFactory.newClient(getHost(), getAppServicesPort(), getModulesDatabaseName(), + getRestAdminUsername(), getRestAdminPassword(), getAppServicesAuthentication(), getAppServicesSslContext(), + getAppServicesSslHostnameVerifier()); + } + /** * Like newDatabaseClient, but connects to schemas database. * * @return */ public DatabaseClient newSchemasDatabaseClient() { - return DatabaseClientFactory.newClient(getHost(), getRestPort(), getSchemasDatabaseName(), - getRestAdminUsername(), getRestAdminPassword(), getRestAuthentication(), getRestSslContext(), - getRestSslHostnameVerifier()); - } - - public StaticChecker newStaticChecker() { - if (isStaticCheckAssets()) { - String xccUri = "xcc://%s:%s@%s:%d"; - xccUri = String.format(xccUri, getRestAdminUsername(), getRestAdminPassword(), getHost(), getRestPort()); - XccStaticChecker checker = new XccStaticChecker(new XccTemplate(xccUri)); - checker.setBulkCheck(isBulkLoadAssets()); - checker.setCheckLibraryModules(isStaticCheckLibraryAssets()); - return checker; - } - return null; - } - - /** - * @return an XccAssetLoader based on the configuration properties in this class - */ - public XccAssetLoader newXccAssetLoader() { - XccAssetLoader l = new XccAssetLoader(); - l.setHost(getHost()); - l.setUsername(getRestAdminUsername()); - l.setPassword(getRestAdminPassword()); - l.setDatabaseName(getModulesDatabaseName()); - if (getAppServicesPort() != null) { - l.setPort(getAppServicesPort()); - } - - String permissions = getModulePermissions(); - if (permissions != null) { - l.setPermissions(permissions); - } - - String[] extensions = getAdditionalBinaryExtensions(); - if (extensions != null) { - DefaultDocumentFormatGetter getter = new DefaultDocumentFormatGetter(); - for (String ext : extensions) { - getter.getBinaryExtensions().add(ext); - } - l.setDocumentFormatGetter(getter); - } - - if (assetFileFilter != null) { - l.setFileFilter(assetFileFilter); - } - - if (isReplaceTokensInModules()) { - l.setModuleTokenReplacer(buildModuleTokenReplacer()); - } - - l.setBulkLoad(isBulkLoadAssets()); - return l; - } - - protected ModuleTokenReplacer buildModuleTokenReplacer() { - DefaultModuleTokenReplacer r = isUseRoxyTokenPrefix() ? new RoxyModuleTokenReplacer() : new DefaultModuleTokenReplacer(); - if (customTokens != null && !customTokens.isEmpty()) { - r.addPropertiesSource(new PropertiesSource() { - @Override - public Properties getProperties() { - Properties p = new Properties(); - p.putAll(customTokens); - return p; - } - }); - } - - if (getModuleTokensPropertiesSources() != null) { - for (PropertiesSource ps : getModuleTokensPropertiesSources()) { - r.addPropertiesSource(ps); - } - } - - return r; + return DatabaseClientFactory.newClient(getHost(), getAppServicesPort(), getSchemasDatabaseName(), + getRestAdminUsername(), getRestAdminPassword(), getAppServicesAuthentication(), getAppServicesSslContext(), + getAppServicesSslHostnameVerifier()); } /** @@ -737,4 +670,28 @@ public boolean isNoRestServer() { public void setNoRestServer(boolean noRestServer) { this.noRestServer = noRestServer; } + + public SSLContext getAppServicesSslContext() { + return appServicesSslContext; + } + + public void setAppServicesSslContext(SSLContext appServicesSslContext) { + this.appServicesSslContext = appServicesSslContext; + } + + public SSLHostnameVerifier getAppServicesSslHostnameVerifier() { + return appServicesSslHostnameVerifier; + } + + public void setAppServicesSslHostnameVerifier(SSLHostnameVerifier appServicesSslHostnameVerifier) { + this.appServicesSslHostnameVerifier = appServicesSslHostnameVerifier; + } + + public Authentication getAppServicesAuthentication() { + return appServicesAuthentication; + } + + public void setAppServicesAuthentication(Authentication appServicesAuthentication) { + this.appServicesAuthentication = appServicesAuthentication; + } } diff --git a/src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java b/src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java index 4cc1bfdf3..064b9baa9 100644 --- a/src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java @@ -23,7 +23,7 @@ public abstract class AbstractCommand extends LoggingObject implements Command { private int executeSortOrder = Integer.MAX_VALUE; private boolean storeResourceIdsAsCustomTokens = false; - protected TokenReplacer tokenReplacer = new DefaultTokenReplacer(); + protected PayloadTokenReplacer payloadTokenReplacer = new DefaultPayloadTokenReplacer(); private FilenameFilter resourceFilenameFilter = new ResourceFilenameFilter(); /** @@ -87,7 +87,7 @@ protected String copyFileToString(File f) { */ protected String copyFileToString(File f, CommandContext context) { String str = copyFileToString(f); - return str != null ? tokenReplacer.replaceTokens(str, context.getAppConfig(), false) : str; + return str != null ? payloadTokenReplacer.replaceTokens(str, context.getAppConfig(), false) : str; } /** @@ -146,8 +146,8 @@ protected File[] listFilesInDirectory(File dir) { return files; } - public void setTokenReplacer(TokenReplacer tokenReplacer) { - this.tokenReplacer = tokenReplacer; + public void setPayloadTokenReplacer(PayloadTokenReplacer payloadTokenReplacer) { + this.payloadTokenReplacer = payloadTokenReplacer; } public void setExecuteSortOrder(int executeSortOrder) { diff --git a/src/main/java/com/marklogic/appdeployer/command/DefaultTokenReplacer.java b/src/main/java/com/marklogic/appdeployer/command/DefaultPayloadTokenReplacer.java similarity index 95% rename from src/main/java/com/marklogic/appdeployer/command/DefaultTokenReplacer.java rename to src/main/java/com/marklogic/appdeployer/command/DefaultPayloadTokenReplacer.java index 84be05976..7968cf187 100644 --- a/src/main/java/com/marklogic/appdeployer/command/DefaultTokenReplacer.java +++ b/src/main/java/com/marklogic/appdeployer/command/DefaultPayloadTokenReplacer.java @@ -4,7 +4,7 @@ import com.marklogic.appdeployer.AppConfig; -public class DefaultTokenReplacer implements TokenReplacer { +public class DefaultPayloadTokenReplacer implements PayloadTokenReplacer { public String replaceTokens(String payload, AppConfig appConfig, boolean isTestResource) { payload = replaceDefaultTokens(payload, appConfig, isTestResource); diff --git a/src/main/java/com/marklogic/appdeployer/command/TokenReplacer.java b/src/main/java/com/marklogic/appdeployer/command/PayloadTokenReplacer.java similarity index 75% rename from src/main/java/com/marklogic/appdeployer/command/TokenReplacer.java rename to src/main/java/com/marklogic/appdeployer/command/PayloadTokenReplacer.java index 792af3927..d4ec1e44a 100644 --- a/src/main/java/com/marklogic/appdeployer/command/TokenReplacer.java +++ b/src/main/java/com/marklogic/appdeployer/command/PayloadTokenReplacer.java @@ -7,7 +7,7 @@ * Typically, the tokens are replaced by values in the AppConfig instance. This allows for configuration files to be * reused across applications with different names. */ -public interface TokenReplacer { +public interface PayloadTokenReplacer { - public String replaceTokens(String payload, AppConfig appConfig, boolean isTestResource); + String replaceTokens(String payload, AppConfig appConfig, boolean isTestResource); } diff --git a/src/main/java/com/marklogic/appdeployer/command/appservers/UpdateRestApiServersCommand.java b/src/main/java/com/marklogic/appdeployer/command/appservers/UpdateRestApiServersCommand.java index 1bbec77fb..dbb14e700 100644 --- a/src/main/java/com/marklogic/appdeployer/command/appservers/UpdateRestApiServersCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/appservers/UpdateRestApiServersCommand.java @@ -38,11 +38,11 @@ public void execute(CommandContext context) { String payload = copyFileToString(f); - String json = tokenReplacer.replaceTokens(payload, appConfig, false); + String json = payloadTokenReplacer.replaceTokens(payload, appConfig, false); mgr.save(json); if (appConfig.isTestPortSet()) { - json = tokenReplacer.replaceTokens(payload, appConfig, true); + json = payloadTokenReplacer.replaceTokens(payload, appConfig, true); mgr.save(json); } } else { diff --git a/src/main/java/com/marklogic/appdeployer/command/clusters/ModifyLocalClusterCommand.java b/src/main/java/com/marklogic/appdeployer/command/clusters/ModifyLocalClusterCommand.java index 46d1aae40..3de0ecade 100644 --- a/src/main/java/com/marklogic/appdeployer/command/clusters/ModifyLocalClusterCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/clusters/ModifyLocalClusterCommand.java @@ -25,7 +25,7 @@ public void execute(CommandContext context) { for (File f : configDir.listFiles()) { if (f.isFile() && f.getName().startsWith("local-cluster")) { String payload = copyFileToString(f); - payload = tokenReplacer.replaceTokens(payload, context.getAppConfig(), false); + payload = payloadTokenReplacer.replaceTokens(payload, context.getAppConfig(), false); new ClusterManager(context.getManageClient()).modifyLocalCluster(payload, context.getAdminManager()); } } diff --git a/src/main/java/com/marklogic/appdeployer/command/databases/DeployContentDatabasesCommand.java b/src/main/java/com/marklogic/appdeployer/command/databases/DeployContentDatabasesCommand.java index 53715290e..824ac4bca 100644 --- a/src/main/java/com/marklogic/appdeployer/command/databases/DeployContentDatabasesCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/databases/DeployContentDatabasesCommand.java @@ -49,7 +49,7 @@ public void execute(CommandContext context) { String payload = getPayload(context); if (payload != null) { DatabaseManager dbMgr = new DatabaseManager(context.getManageClient()); - String json = tokenReplacer.replaceTokens(payload, appConfig, true); + String json = payloadTokenReplacer.replaceTokens(payload, appConfig, true); SaveReceipt receipt = dbMgr.save(json); if (shouldCreateForests(context, payload)) { buildDeployForestsCommand(payload, receipt, context).execute(context); @@ -74,12 +74,12 @@ public void undo(CommandContext context) { if (node != null) { logger.info("No content database files found, so not deleting content databases"); String payload = node.toString(); - String json = tokenReplacer.replaceTokens(payload, appConfig, false); + String json = payloadTokenReplacer.replaceTokens(payload, appConfig, false); DatabaseManager dbMgr = newDatabaseManageForDeleting(context); dbMgr.delete(json); if (appConfig.isTestPortSet()) { - json = tokenReplacer.replaceTokens(payload, appConfig, true); + json = payloadTokenReplacer.replaceTokens(payload, appConfig, true); dbMgr.delete(json); } } else { diff --git a/src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java b/src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java index 848cf15e2..ae2bc8238 100644 --- a/src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java @@ -127,19 +127,19 @@ protected String getForestDeleteLevel(AppConfig appConfig) { /** * Builds the XML or JSON payload for this command, based on the given CommandContext. - * + * * @param context * @return */ public String buildPayload(CommandContext context) { String payload = getPayload(context); - return payload != null ? tokenReplacer.replaceTokens(payload, context.getAppConfig(), false) : null; + return payload != null ? payloadTokenReplacer.replaceTokens(payload, context.getAppConfig(), false) : null; } /** * Get the payload based on the given CommandContext. Only loads the payload, does not replace any tokens in it. * Call buildPayload to construct a payload with all tokens replaced. - * + * * @param context * @return */ @@ -196,7 +196,7 @@ protected boolean customForestsExist(CommandContext context, String dbName) { /** * Allows for how an instance of DeployForestsCommand is built to be overridden by a subclass. - * + * * @param dbPayload * Needed so we can look up forest counts based on the database name * @param receipt @@ -217,7 +217,7 @@ protected DeployForestsCommand buildDeployForestsCommand(String dbPayload, SaveR /** * Checks the forestCounts map in AppConfig to see if the client has specified a number of forests per host for this * database. - * + * * @param dbPayload * @param context * @return diff --git a/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java b/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java index 5a3fe9790..4e1e87e08 100644 --- a/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java @@ -92,7 +92,7 @@ protected void createForests(String originalPayload, CommandContext context) { for (int i = countOfExistingForests + 1; i <= desiredNumberOfForests;) { for (String hostName : hostNames) { if (i <= desiredNumberOfForests) { - String payload = tokenReplacer.replaceTokens(originalPayload, appConfig, false); + String payload = payloadTokenReplacer.replaceTokens(originalPayload, appConfig, false); payload = payload.replace("%%FOREST_HOST%%", hostName); String forestName = getForestName(appConfig, i); payload = payload.replace("%%FOREST_NAME%%", forestName); diff --git a/src/main/java/com/marklogic/appdeployer/command/modules/DefaultModulesLoaderFactory.java b/src/main/java/com/marklogic/appdeployer/command/modules/DefaultModulesLoaderFactory.java new file mode 100644 index 000000000..280d4dfa9 --- /dev/null +++ b/src/main/java/com/marklogic/appdeployer/command/modules/DefaultModulesLoaderFactory.java @@ -0,0 +1,98 @@ +package com.marklogic.appdeployer.command.modules; + +import com.marklogic.appdeployer.AppConfig; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.ext.tokenreplacer.DefaultTokenReplacer; +import com.marklogic.client.ext.tokenreplacer.PropertiesSource; +import com.marklogic.client.ext.tokenreplacer.RoxyTokenReplacer; +import com.marklogic.client.ext.tokenreplacer.TokenReplacer; +import com.marklogic.client.helper.LoggingObject; +import com.marklogic.client.modulesloader.ModulesLoader; +import com.marklogic.client.modulesloader.ModulesManager; +import com.marklogic.client.modulesloader.impl.*; +import com.marklogic.xcc.template.XccTemplate; + +import java.io.File; +import java.util.Map; +import java.util.Properties; + +public class DefaultModulesLoaderFactory extends LoggingObject implements ModulesLoaderFactory { + + @Override + public ModulesLoader newModulesLoader(AppConfig appConfig) { + ModulesManager modulesManager = null; + String path = appConfig.getModuleTimestampsPath(); + if (path != null) { + modulesManager = new PropertiesModuleManager(new File(path)); + } + + DatabaseClient modulesDatabaseClient = appConfig.newModulesDatabaseClient(); + AssetFileLoader assetFileLoader = new AssetFileLoader(modulesDatabaseClient, modulesManager); + + String permissions = appConfig.getModulePermissions(); + if (permissions != null) { + assetFileLoader.setPermissions(permissions); + } + + String[] extensions = appConfig.getAdditionalBinaryExtensions(); + if (extensions != null) { + assetFileLoader.setAdditionalBinaryExtensions(extensions); + } + + if (appConfig.getAssetFileFilter() != null) { + assetFileLoader.addFileFilter(appConfig.getAssetFileFilter()); + } + + if (appConfig.isReplaceTokensInModules()) { + assetFileLoader.setTokenReplacer(buildModuleTokenReplacer(appConfig)); + } + + DefaultModulesLoader modulesLoader = new DefaultModulesLoader(assetFileLoader); + modulesLoader.setModulesManager(modulesManager); + + if (appConfig.isStaticCheckAssets()) { + modulesLoader.setStaticChecker(newStaticChecker(appConfig)); + } + return modulesLoader; + } + + /** + * Currently only have an XCC implementation for static checking, as XCC gives much more useful error messages + * than REST does. + * + * @param appConfig + * @return + */ + protected StaticChecker newStaticChecker(AppConfig appConfig) { + String xccUri = "xcc://%s:%s@%s:%d"; + xccUri = String.format(xccUri, appConfig.getRestAdminUsername(), appConfig.getRestAdminPassword(), + appConfig.getHost(), appConfig.getRestPort()); + XccStaticChecker checker = new XccStaticChecker(new XccTemplate(xccUri)); + checker.setBulkCheck(appConfig.isBulkLoadAssets()); + checker.setCheckLibraryModules(appConfig.isStaticCheckLibraryAssets()); + return checker; + } + + protected TokenReplacer buildModuleTokenReplacer(AppConfig appConfig) { + DefaultTokenReplacer r = appConfig.isUseRoxyTokenPrefix() ? new RoxyTokenReplacer() : new DefaultTokenReplacer(); + final Map customTokens = appConfig.getCustomTokens(); + if (customTokens != null && !customTokens.isEmpty()) { + r.addPropertiesSource(new PropertiesSource() { + @Override + public Properties getProperties() { + Properties p = new Properties(); + p.putAll(customTokens); + return p; + } + }); + } + + if (appConfig.getModuleTokensPropertiesSources() != null) { + for (PropertiesSource ps : appConfig.getModuleTokensPropertiesSources()) { + r.addPropertiesSource(ps); + } + } + + return r; + } +} diff --git a/src/main/java/com/marklogic/appdeployer/command/modules/LoadModulesCommand.java b/src/main/java/com/marklogic/appdeployer/command/modules/LoadModulesCommand.java index d12a35527..561963b6d 100644 --- a/src/main/java/com/marklogic/appdeployer/command/modules/LoadModulesCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/modules/LoadModulesCommand.java @@ -1,16 +1,15 @@ package com.marklogic.appdeployer.command.modules; -import java.io.File; - -import com.marklogic.client.DatabaseClient; -import com.marklogic.client.modulesloader.ModulesLoader; -import com.marklogic.client.modulesloader.impl.DefaultModulesLoader; -import com.marklogic.client.modulesloader.impl.PropertiesModuleManager; -import com.marklogic.client.modulesloader.impl.TestServerModulesFinder; import com.marklogic.appdeployer.AppConfig; import com.marklogic.appdeployer.command.AbstractCommand; import com.marklogic.appdeployer.command.CommandContext; import com.marklogic.appdeployer.command.SortOrderConstants; +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.modulesloader.ModulesLoader; +import com.marklogic.client.modulesloader.impl.DefaultModulesLoader; +import com.marklogic.client.modulesloader.impl.TestServerModulesFinder; + +import java.io.File; /** * Command for loading modules via an instance of DefaultModulesLoader, which depends on an instance of XccAssetLoader - @@ -18,100 +17,104 @@ */ public class LoadModulesCommand extends AbstractCommand { - private ModulesLoader modulesLoader; - - public LoadModulesCommand() { - setExecuteSortOrder(SortOrderConstants.LOAD_MODULES); - } - - /** - * Public so that a client can initialize the ModulesLoader and then access it via the getter; this is useful for a - * tool like ml-gradle, where the ModulesLoader can be reused by multiple tasks. - * - * @param context - */ - public void initializeDefaultModulesLoader(CommandContext context) { - logger.info("Initializing instance of DefaultModulesLoader"); - DefaultModulesLoader l = new DefaultModulesLoader(context.getAppConfig().newXccAssetLoader()); - String path = context.getAppConfig().getModuleTimestampsPath(); - if (path != null) { - l.setModulesManager(new PropertiesModuleManager(new File(path))); + private ModulesLoader modulesLoader; + private ModulesLoaderFactory modulesLoaderFactory; + + public LoadModulesCommand() { + setExecuteSortOrder(SortOrderConstants.LOAD_MODULES); + this.modulesLoaderFactory = new DefaultModulesLoaderFactory(); + } + + /** + * Public so that a client can initialize the ModulesLoader and then access it via the getter; this is useful for a + * tool like ml-gradle, where the ModulesLoader can be reused by multiple tasks. + * + * @param context + */ + public void initializeDefaultModulesLoader(CommandContext context) { + logger.info("Initializing new instance of ModulesLoader"); + this.modulesLoader = modulesLoaderFactory.newModulesLoader(context.getAppConfig()); + } + + @Override + public void execute(CommandContext context) { + loadModulesIntoMainServer(context); + + if (context.getAppConfig().isTestPortSet()) { + loadModulesIntoTestServer(context); + } + } + + /** + * If we have multiple module paths, we want to load via XCC the assets for each first, and then iterate over the + * paths again and load all the REST API resources. This ensures that if the REST server for loading REST API + * resources has a custom rewriter, it's guaranteed to be loaded before we try to load any REST API resources. + * + * @param context + */ + protected void loadModulesIntoMainServer(CommandContext context) { + if (modulesLoader == null) { + initializeDefaultModulesLoader(context); + } + + AppConfig config = context.getAppConfig(); + DatabaseClient client = config.newDatabaseClient(); + + try { + for (String modulesPath : config.getModulePaths()) { + logger.info("Loading asset modules from dir: " + modulesPath); + modulesLoader.loadModules(new File(modulesPath), new AssetModulesFinder(), client); + } + + for (String modulesPath : config.getModulePaths()) { + logger.info("Loading all non-asset modules from dir: " + modulesPath); + modulesLoader.loadModules(new File(modulesPath), new AllButAssetsModulesFinder(), client); + } + } finally { + client.release(); + } + } + + /** + * We use a customized impl of DefaultModulesLoader here so we can ensure that options are always loaded again into + * the test server. + * + * @param context + */ + protected void loadModulesIntoTestServer(CommandContext context) { + AppConfig config = context.getAppConfig(); + DatabaseClient client = config.newTestDatabaseClient(); + ModulesLoader testLoader = buildTestModulesLoader(context); + try { + for (String modulesPath : config.getModulePaths()) { + logger.info("Loading modules into test server from dir: " + modulesPath); + testLoader.loadModules(new File(modulesPath), new TestServerModulesFinder(), client); + } + } finally { + client.release(); } - l.setStaticChecker(context.getAppConfig().newStaticChecker()); - this.modulesLoader = l; - } - - @Override - public void execute(CommandContext context) { - loadModulesIntoMainServer(context); - - if (context.getAppConfig().isTestPortSet()) { - loadModulesIntoTestServer(context); - } - } - - /** - * If we have multiple module paths, we want to load via XCC the assets for each first, and then iterate over the - * paths again and load all the REST API resources. This ensures that if the REST server for loading REST API - * resources has a custom rewriter, it's guaranteed to be loaded before we try to load any REST API resources. - * - * @param context - */ - protected void loadModulesIntoMainServer(CommandContext context) { - if (modulesLoader == null) { - initializeDefaultModulesLoader(context); - } - - AppConfig config = context.getAppConfig(); - DatabaseClient client = config.newDatabaseClient(); - - try { - for (String modulesPath : config.getModulePaths()) { - logger.info("Loading asset modules from dir: " + modulesPath); - modulesLoader.loadModules(new File(modulesPath), new AssetModulesFinder(), client); - } - - for (String modulesPath : config.getModulePaths()) { - logger.info("Loading all non-asset modules from dir: " + modulesPath); - modulesLoader.loadModules(new File(modulesPath), new AllButAssetsModulesFinder(), client); - } - } finally { - client.release(); - } - } - - /** - * We use a customized impl of DefaultModulesLoader here so we can ensure that options are always loaded again into - * the test server. - * - * @param context - */ - protected void loadModulesIntoTestServer(CommandContext context) { - AppConfig config = context.getAppConfig(); - DatabaseClient client = config.newTestDatabaseClient(); - ModulesLoader testLoader = buildTestModulesLoader(context); - try { - for (String modulesPath : config.getModulePaths()) { - logger.info("Loading modules into test server from dir: " + modulesPath); - testLoader.loadModules(new File(modulesPath), new TestServerModulesFinder(), client); - } - } finally { - client.release(); - } - } - - protected ModulesLoader buildTestModulesLoader(CommandContext context) { - // Don't need an asset loader here, as only options/properties are loaded for the test server - DefaultModulesLoader l = new DefaultModulesLoader(); - l.setModulesManager(null); - return l; - } - - public void setModulesLoader(ModulesLoader modulesLoader) { - this.modulesLoader = modulesLoader; - } - - public ModulesLoader getModulesLoader() { - return modulesLoader; - } + } + + protected ModulesLoader buildTestModulesLoader(CommandContext context) { + // Don't need an asset loader here, as only options/properties are loaded for the test server + DefaultModulesLoader l = new DefaultModulesLoader(); + l.setModulesManager(null); + return l; + } + + public void setModulesLoader(ModulesLoader modulesLoader) { + this.modulesLoader = modulesLoader; + } + + public ModulesLoader getModulesLoader() { + return modulesLoader; + } + + public void setModulesLoaderFactory(ModulesLoaderFactory modulesLoaderFactory) { + this.modulesLoaderFactory = modulesLoaderFactory; + } + + public ModulesLoaderFactory getModulesLoaderFactory() { + return modulesLoaderFactory; + } } diff --git a/src/main/java/com/marklogic/appdeployer/command/modules/ModulesLoaderFactory.java b/src/main/java/com/marklogic/appdeployer/command/modules/ModulesLoaderFactory.java new file mode 100644 index 000000000..2266177f3 --- /dev/null +++ b/src/main/java/com/marklogic/appdeployer/command/modules/ModulesLoaderFactory.java @@ -0,0 +1,13 @@ +package com.marklogic.appdeployer.command.modules; + +import com.marklogic.appdeployer.AppConfig; +import com.marklogic.client.modulesloader.ModulesLoader; + +/** + * Interface for objects that can construct a ModulesLoader based on the configuration information in the given + * AppConfig instance. + */ +public interface ModulesLoaderFactory { + + ModulesLoader newModulesLoader(AppConfig appConfig); +} diff --git a/src/main/java/com/marklogic/appdeployer/command/restapis/DeployRestApiServersCommand.java b/src/main/java/com/marklogic/appdeployer/command/restapis/DeployRestApiServersCommand.java index 1eaaaffd6..d046a2338 100644 --- a/src/main/java/com/marklogic/appdeployer/command/restapis/DeployRestApiServersCommand.java +++ b/src/main/java/com/marklogic/appdeployer/command/restapis/DeployRestApiServersCommand.java @@ -57,10 +57,10 @@ public void execute(CommandContext context) { RestApiManager mgr = new RestApiManager(context.getManageClient()); AppConfig appConfig = context.getAppConfig(); - mgr.createRestApi(tokenReplacer.replaceTokens(payload, appConfig, false)); + mgr.createRestApi(payloadTokenReplacer.replaceTokens(payload, appConfig, false)); if (appConfig.isTestPortSet()) { - mgr.createRestApi(tokenReplacer.replaceTokens(payload, appConfig, true)); + mgr.createRestApi(payloadTokenReplacer.replaceTokens(payload, appConfig, true)); } } } @@ -123,7 +123,7 @@ protected void deleteMainRestServer(CommandContext context) { String payload = getRestApiPayload(context); if (payload != null) { - payload = tokenReplacer.replaceTokens(payload, appConfig, false); + payload = payloadTokenReplacer.replaceTokens(payload, appConfig, false); final String serverName = new RestApiManager(manageClient).extractNameFromJson(payload); if (mgr.exists(serverName)) { diff --git a/src/main/java/com/marklogic/appdeployer/util/ModulesWatcher.java b/src/main/java/com/marklogic/appdeployer/util/ModulesWatcher.java index a93d35202..fd66f7c65 100644 --- a/src/main/java/com/marklogic/appdeployer/util/ModulesWatcher.java +++ b/src/main/java/com/marklogic/appdeployer/util/ModulesWatcher.java @@ -1,19 +1,20 @@ package com.marklogic.appdeployer.util; -import java.io.File; -import java.util.List; - +import com.marklogic.appdeployer.AppConfig; +import com.marklogic.appdeployer.DefaultAppConfigFactory; +import com.marklogic.appdeployer.command.modules.DefaultModulesLoaderFactory; +import com.marklogic.appdeployer.command.modules.ModulesLoaderFactory; import com.marklogic.client.DatabaseClient; import com.marklogic.client.helper.LoggingObject; import com.marklogic.client.modulesloader.ModulesFinder; +import com.marklogic.client.modulesloader.ModulesLoader; import com.marklogic.client.modulesloader.impl.DefaultModulesFinder; import com.marklogic.client.modulesloader.impl.DefaultModulesLoader; -import com.marklogic.client.modulesloader.impl.PropertiesModuleManager; -import com.marklogic.client.modulesloader.impl.XccAssetLoader; -import com.marklogic.appdeployer.AppConfig; -import com.marklogic.appdeployer.DefaultAppConfigFactory; import com.marklogic.mgmt.util.SystemPropertySource; +import java.io.File; +import java.util.List; + /** * This is a hacked together prototype of loading modules from within groovysh. The idea is that all the necessary * configuration for loading modules can be collected from system properties, which can be set by a tool like ml-gradle. @@ -22,46 +23,51 @@ */ public class ModulesWatcher extends LoggingObject implements Runnable { - private long sleepTime = 1000; + private long sleepTime = 1000; + private AppConfig appConfig; + private ModulesLoaderFactory modulesLoaderFactory; + + public ModulesWatcher(AppConfig appConfig) { + this.appConfig = appConfig; + this.modulesLoaderFactory = new DefaultModulesLoaderFactory(); + } - private AppConfig appConfig; + public static void startFromSystemProps() { + ModulesWatcher mw = new ModulesWatcher(new DefaultAppConfigFactory(new SystemPropertySource()).newAppConfig()); + new Thread(mw).start(); + } - public ModulesWatcher(AppConfig appConfig) { - this.appConfig = appConfig; - } + @Override + public void run() { + ModulesLoader loader = modulesLoaderFactory.newModulesLoader(appConfig); + if (loader instanceof DefaultModulesLoader) { + ((DefaultModulesLoader) loader).setCatchExceptions(true); + } - public static void startFromSystemProps() { - ModulesWatcher mw = new ModulesWatcher(new DefaultAppConfigFactory(new SystemPropertySource()).newAppConfig()); - new Thread(mw).start(); - } + DatabaseClient client = appConfig.newDatabaseClient(); + List paths = appConfig.getModulePaths(); + ModulesFinder finder = new DefaultModulesFinder(); + while (true) { + for (String modulesPath : paths) { + loader.loadModules(new File(modulesPath), finder, client); + } + try { + Thread.sleep(sleepTime); + } catch (InterruptedException ie) { + // Ignore + } + } + } - @Override - public void run() { - XccAssetLoader xal = appConfig.newXccAssetLoader(); - DefaultModulesLoader loader = new DefaultModulesLoader(xal); - String path = appConfig.getModuleTimestampsPath(); - if (path != null) { - loader.setModulesManager(new PropertiesModuleManager(new File(path))); - } - loader.setStaticChecker(appConfig.newStaticChecker()); - loader.setCatchExceptions(true); - DatabaseClient client = appConfig.newDatabaseClient(); - List paths = appConfig.getModulePaths(); - ModulesFinder finder = new DefaultModulesFinder(); - while (true) { - for (String modulesPath : paths) { - loader.loadModules(new File(modulesPath), finder, client); - } - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ie) { - // Ignore - } - } - } + public void setSleepTime(long sleepTime) { + this.sleepTime = sleepTime; + } - public void setSleepTime(long sleepTime) { - this.sleepTime = sleepTime; - } + public ModulesLoaderFactory getModulesLoaderFactory() { + return modulesLoaderFactory; + } + public void setModulesLoaderFactory(ModulesLoaderFactory modulesLoaderFactory) { + this.modulesLoaderFactory = modulesLoaderFactory; + } } diff --git a/src/test/java/com/marklogic/appdeployer/AbstractAppDeployerTest.java b/src/test/java/com/marklogic/appdeployer/AbstractAppDeployerTest.java index 244f1dfa7..5575c1602 100644 --- a/src/test/java/com/marklogic/appdeployer/AbstractAppDeployerTest.java +++ b/src/test/java/com/marklogic/appdeployer/AbstractAppDeployerTest.java @@ -1,6 +1,7 @@ package com.marklogic.appdeployer; import com.marklogic.appdeployer.command.Command; +import com.marklogic.appdeployer.command.modules.DefaultModulesLoaderFactory; import com.marklogic.appdeployer.command.modules.LoadModulesCommand; import com.marklogic.appdeployer.command.restapis.DeployRestApiServersCommand; import com.marklogic.appdeployer.impl.SimpleAppDeployer; @@ -80,8 +81,8 @@ protected XccTemplate newModulesXccTemplate() { */ protected LoadModulesCommand buildLoadModulesCommand() { LoadModulesCommand command = new LoadModulesCommand(); - DefaultModulesLoader loader = new DefaultModulesLoader(appConfig.newXccAssetLoader()); - loader.setStaticChecker(appConfig.newStaticChecker()); + appConfig.setModuleTimestampsPath(null); + DefaultModulesLoader loader = (DefaultModulesLoader)(new DefaultModulesLoaderFactory().newModulesLoader(appConfig)); loader.setModulesManager(null); command.setModulesLoader(loader); return command; diff --git a/src/test/java/com/marklogic/appdeployer/command/es/GenerateModelArtifactsTest.java b/src/test/java/com/marklogic/appdeployer/command/es/GenerateModelArtifactsTest.java index 84b74e492..112278640 100644 --- a/src/test/java/com/marklogic/appdeployer/command/es/GenerateModelArtifactsTest.java +++ b/src/test/java/com/marklogic/appdeployer/command/es/GenerateModelArtifactsTest.java @@ -20,7 +20,7 @@ public class GenerateModelArtifactsTest extends AbstractAppDeployerTest { public void tearDown() { initializeAppDeployer(new DeployContentDatabasesCommand(), new DeploySchemasDatabaseCommand(), new DeployRestApiServersCommand()); - //undeploySampleApp(); + undeploySampleApp(); } @Test diff --git a/src/test/java/com/marklogic/appdeployer/command/modules/LoadModulesTest.java b/src/test/java/com/marklogic/appdeployer/command/modules/LoadModulesTest.java index 42d3eb4d7..1a663c103 100644 --- a/src/test/java/com/marklogic/appdeployer/command/modules/LoadModulesTest.java +++ b/src/test/java/com/marklogic/appdeployer/command/modules/LoadModulesTest.java @@ -1,19 +1,16 @@ package com.marklogic.appdeployer.command.modules; -import java.io.File; - -import com.marklogic.appdeployer.command.CommandContext; -import com.marklogic.client.modulesloader.impl.DefaultModulesLoader; -import com.marklogic.junit.Fragment; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - import com.marklogic.appdeployer.AbstractAppDeployerTest; import com.marklogic.appdeployer.command.restapis.DeployRestApiServersCommand; import com.marklogic.client.modulesloader.impl.AssetFileFilter; +import com.marklogic.junit.Fragment; import com.marklogic.junit.PermissionsFragment; import com.marklogic.xcc.template.XccTemplate; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; public class LoadModulesTest extends AbstractAppDeployerTest { @@ -83,8 +80,8 @@ public void loadModulesFromMultiplePaths() { "/ext/some-lib.xqy"); } - @Test - public void loadModulesWithCustomPermissions() { + @Test + public void loadModulesWithCustomPermissions() { appConfig.setModulePermissions(appConfig.getModulePermissions() + ",app-user,execute"); initializeAppDeployer(new DeployRestApiServersCommand(true), buildLoadModulesCommand()); @@ -92,12 +89,19 @@ public void loadModulesWithCustomPermissions() { appDeployer.deploy(appConfig); PermissionsFragment perms = getDocumentPermissions("/ext/sample-lib.xqy", xccTemplate); - perms.prettyPrint(); - perms.assertPermissionCount(4); + perms.assertPermissionCount(6); + + // Default permissions set by AppConfig perms.assertPermissionExists("rest-admin", "read"); perms.assertPermissionExists("rest-admin", "update"); perms.assertPermissionExists("rest-extension-user", "execute"); + + // Custom permission perms.assertPermissionExists("app-user", "execute"); + + // Permissions that the REST API still applies, which seems like a bug + perms.assertPermissionExists("rest-reader", "read"); + perms.assertPermissionExists("rest-writer", "update"); } @Test @@ -148,13 +152,20 @@ private void assertModuleExistsWithDefaultPermissions(String message, String uri /** * Apparently, the REST API won't let you remove these 3 default permissions, they're always present. + * + * And, now that we're loading modules via the REST API by default, rest-reader/read and rest-writer/update are + * always present, at least on 8.0-6.3 and 9.0-1.1, which seems like a bug. */ private void assertDefaultPermissionsExists(String uri) { PermissionsFragment perms = getDocumentPermissions(uri, xccTemplate); - perms.assertPermissionCount(3); + perms.assertPermissionCount(5); perms.assertPermissionExists("rest-admin", "read"); perms.assertPermissionExists("rest-admin", "update"); perms.assertPermissionExists("rest-extension-user", "execute"); + + // Not really expected! + perms.assertPermissionExists("rest-reader", "read"); + perms.assertPermissionExists("rest-writer", "update"); } } diff --git a/src/test/java/com/marklogic/appdeployer/command/restapis/CreateRestApiAsNonAdminUserTest.java b/src/test/java/com/marklogic/appdeployer/command/restapis/CreateRestApiAsNonAdminUserTest.java index 9e1020917..a6dde08f7 100644 --- a/src/test/java/com/marklogic/appdeployer/command/restapis/CreateRestApiAsNonAdminUserTest.java +++ b/src/test/java/com/marklogic/appdeployer/command/restapis/CreateRestApiAsNonAdminUserTest.java @@ -58,9 +58,13 @@ public void test() { // And now ensure that the module was loaded correctly PermissionsFragment perms = getDocumentPermissions("/ext/hello-lib.xqy", xccTemplate); - perms.assertPermissionCount(3); + perms.assertPermissionCount(5); perms.assertPermissionExists("rest-admin", "read"); perms.assertPermissionExists("rest-admin", "update"); perms.assertPermissionExists("rest-extension-user", "execute"); + + // Non-expected permissions that ML, as of 9.0-1.1, still adds by default + perms.assertPermissionExists("rest-reader", "read"); + perms.assertPermissionExists("rest-writer", "update"); } } diff --git a/src/test/java/com/marklogic/appdeployer/export/ExportServerTest.java b/src/test/java/com/marklogic/appdeployer/export/ExportServerTest.java index fc039d25e..b62388469 100644 --- a/src/test/java/com/marklogic/appdeployer/export/ExportServerTest.java +++ b/src/test/java/com/marklogic/appdeployer/export/ExportServerTest.java @@ -13,7 +13,7 @@ public class ExportServerTest extends AbstractExportTest { @After public void teardown() { - //undeploySampleApp(); + undeploySampleApp(); } @Test diff --git a/src/test/resources/non-admin-test/ml-config/security/roles/sample-app-role.json b/src/test/resources/non-admin-test/ml-config/security/roles/sample-app-role.json index f894c4783..1534f170b 100644 --- a/src/test/resources/non-admin-test/ml-config/security/roles/sample-app-role.json +++ b/src/test/resources/non-admin-test/ml-config/security/roles/sample-app-role.json @@ -1,10 +1,10 @@ { "role-name" : "sample-app-role", - "role" : [ "rest-admin" ], + "role" : [ "rest-admin", "rest-writer", "rest-reader" ], "privilege": [ { - "privilege-name": "xdbc:insert-in", - "action": "http://marklogic.com/xdmp/privileges/xdbc-insert-in", + "privilege-name": "xdmp:eval-in", + "action": "http://marklogic.com/xdmp/privileges/xdmp-eval-in", "kind": "execute" }, { @@ -13,4 +13,4 @@ "kind": "execute" } ] -} \ No newline at end of file +}