Merge pull request #7337 from geonetwork/ha

Improvements to allow future deployment of multiple instances

common/src/main/java/org/fao/geonet/utils/Env.java

@@ -0,0 +1,56 @@
+//=============================================================================
+//===	Copyright (C) 2001-2023 Food and Agriculture Organization of the
+//===	United Nations (FAO-UN), United Nations World Food Programme (WFP)
+//===	and United Nations Environment Programme (UNEP)
+//===
+//===	This program is free software; you can redistribute it and/or modify
+//===	it under the terms of the GNU General Public License as published by
+//===	the Free Software Foundation; either version 2 of the License, or (at
+//===	your option) any later version.
+//===
+//===	This program is distributed in the hope that it will be useful, but
+//===	WITHOUT ANY WARRANTY; without even the implied warranty of
+//===	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//===	General Public License for more details.
+//===
+//===	You should have received a copy of the GNU General Public License
+//===	along with this program; if not, write to the Free Software
+//===	Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//===
+//===	Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+//===	Rome - Italy. email: [email protected]
+//==============================================================================
+
+package org.fao.geonet.utils;
+
+import org.apache.commons.lang.StringUtils;
+
+public final class Env {
+
+    /**
+     * Retrieves an environment variable with this priority:
+     * - Java environment variable.
+     * - System environment variable.
+     * - Default value provided as parameter.
+     *
+     * @param propertyName
+     * @param defaultValue
+     * @return
+     */
+    public static String getPropertyFromEnv(String propertyName, String defaultValue) {
+        // Check if provided in Java environment variable
+        String propertyValue = System.getProperty(propertyName);
+
+        if (StringUtils.isEmpty(propertyValue)) {
+            // System environment variable
+            propertyValue = System.getenv(propertyName.toUpperCase().replace('.', '_'));
+        }
+
+        if (StringUtils.isEmpty(propertyValue)) {
+            propertyValue = defaultValue;
+        }
+
+        return propertyValue;
+    }
+
+}

core/src/main/java/org/fao/geonet/constants/Geonet.java

@@ -535,6 +535,7 @@ public static final class SortBy {
      */
     public static final class Config {
         public static final String HTMLCACHE_DIR = "htmlCacheDir";
+        public static final String SCHEMAPUBLICATION_DIR = "schemaPublicationDir";
         public static final String INDEX_CONFIG_DIR = "indexConfigDir";
         /**
          * Profiles of languages for autodetection using https://code.google.com/p/language-detection/.

core/src/main/java/org/fao/geonet/kernel/GeonetworkDataDirectory.java

@@ -74,6 +74,7 @@ public class GeonetworkDataDirectory {
     private Path metadataRevisionDir;
     private Path resourcesDir;
     private Path htmlCacheDir;
+    private Path schemaPublicationDir;
     private Path uploadDir;
     private Path formatterDir;
     private Path nodeLessFiles;
@@ -302,10 +303,12 @@ private Path setDataDirectory(JeevesServlet jeevesServlet, String webappName,
         );
         formatterDir = setDir(jeevesServlet, webappName, handlerConfig, formatterDir,
             ".formatter" + KEY_SUFFIX, Geonet.Config.FORMATTER_PATH, "data", "formatter");
-
         htmlCacheDir = setDir(jeevesServlet, webappName, handlerConfig, htmlCacheDir,
             ".htmlcache" + KEY_SUFFIX, Geonet.Config.HTMLCACHE_DIR, handlerConfig.getValue(Geonet.Config.RESOURCES_DIR), "htmlcache"
         );
+        schemaPublicationDir = setDir(jeevesServlet, webappName, handlerConfig, schemaPublicationDir,
+            ".schemapublication" + KEY_SUFFIX, Geonet.Config.SCHEMAPUBLICATION_DIR, handlerConfig.getValue(Geonet.Config.RESOURCES_DIR), "schemapublication"
+        );
         backupDir = setDir(jeevesServlet, webappName, handlerConfig, backupDir,
             ".backup" + KEY_SUFFIX, Geonet.Config.BACKUP_DIR, "data", "backup"
         );
@@ -719,6 +722,23 @@ public void setHtmlCacheDir(Path htmlCacheDir) {
         this.htmlCacheDir = htmlCacheDir;
     }
 
+    /**
+     * Get directory for publishing schemas and making them publicly available.
+     *
+     * @return directory for publishing schemas and making them publicly available.
+     */
+    public Path getSchemaPublicationDir() {
+        return schemaPublicationDir;
+    }
+
+    /**
+     * Set directory for publishing schemas and making them publicly available.
+      * @param schemaPublicationDir
+     */
+    public void setSchemaPublicationDir(Path schemaPublicationDir) {
+        this.schemaPublicationDir = schemaPublicationDir;
+    }
+
     /**
      * Get directory for caching where uploaded files go.
      *

core/src/main/java/org/fao/geonet/kernel/SchemaManager.java

@@ -114,6 +114,7 @@ public class SchemaManager {
     private String defaultSchema;
     private Path basePath;
     private Path resourcePath;
+    private Path schemaPublicationDir;
     private int numberOfCoreSchemasAdded = 0;
 
     public static Path registerXmlCatalogFiles(Path webappDir, Path schemapluginUriCatalog) {
@@ -174,6 +175,7 @@ public static SchemaPlugin getSchemaPlugin(String schemaIdentifier) {
     public void configureFrom(SchemaManager schemaManager, Path basePath, GeonetworkDataDirectory dataDir) {
         this.basePath = basePath;
         this.resourcePath = dataDir.getResourcesDir();
+        this.schemaPublicationDir = dataDir.getSchemaPublicationDir();
         this.schemaPluginsDir = dataDir.getSchemaPluginsDir();
         this.schemaPluginsCat = schemaPluginsDir.resolve("schemaplugin-uri-catalog.xml");
         this.defaultLang = schemaManager.defaultLang;
@@ -200,19 +202,29 @@ private void addResolverRewriteDirectives(GeonetworkDataDirectory dataDir) {
     /**
      * initialize and configure schema manager. should only be on startup.
      *
-     * @param basePath         the web app base path
-     * @param schemaPluginsCat the schema catalogue file
-     * @param sPDir            the schema plugin directory
-     * @param defaultLang      the default language (taken from context)
-     * @param defaultSchema    the default schema (taken from config.xml)
-     */
-    public void configure(ApplicationContext applicationContext, Path basePath, Path resourcePath, Path schemaPluginsCat,
-                          Path sPDir, String defaultLang, String defaultSchema, boolean createOrUpdateSchemaCatalog) throws Exception {
+     * @param basePath               the web app base path
+     * @param resourcePath           the resource folder (eg. images, logo)
+     * @param schemaPublicationDir   the schema publication folder (ie. schema plugin XSDs)
+     * @param schemaPluginsCat       the schema catalogue file (ie. schemaplugin-uri-catalog.xml)
+     * @param sPDir                  the schema plugin directory
+     * @param defaultLang            the default language (taken from context)
+     * @param defaultSchema          the default schema (taken from config.xml)
+     */
+    public void configure(ApplicationContext applicationContext,
+                          Path basePath,
+                          Path resourcePath,
+                          Path schemaPublicationDir,
+                          Path schemaPluginsCat,
+                          Path sPDir,
+                          String defaultLang,
+                          String defaultSchema,
+                          boolean createOrUpdateSchemaCatalog) throws Exception {
 
         hmSchemas.clear();
 
         this.basePath = basePath;
         this.resourcePath = resourcePath;
+        this.schemaPublicationDir = schemaPublicationDir;
         this.schemaPluginsDir = sPDir;
         this.schemaPluginsCat = schemaPluginsCat;
         this.defaultLang = defaultLang;
@@ -1259,7 +1271,7 @@ private void removeSchemaDir(Path schemaDir, String name) {
         }
         deleteDir(schemaDir);
 
-        Path pubSchemaDir = resourcePath.resolve(Geonet.Path.SCHEMAS).resolve(name);
+        Path pubSchemaDir = schemaPublicationDir.resolve(Geonet.Path.SCHEMAS).resolve(name);
         if (Log.isDebugEnabled(Geonet.SCHEMA_MANAGER)) {
             Log.debug(Geonet.SCHEMA_MANAGER, "Removing published schemas directory " + pubSchemaDir);
         }
@@ -1869,7 +1881,7 @@ public String getDefaultSchema() {
      * @param schemaPluginDir the directory containing the schema plugin
      */
     private void copySchemaXSDsToWebApp(String name, Path schemaPluginDir) throws Exception {
-        Path schemasDir = resourcePath.resolve(Geonet.Path.SCHEMAS);
+        Path schemasDir = schemaPublicationDir.resolve(Geonet.Path.SCHEMAS);
         Files.createDirectories(schemasDir);
 
         Path webAppDirSchemaXSD = schemasDir.resolve(name);

core/src/main/java/org/fao/geonet/resources/ResourceFilter.java

@@ -24,23 +24,19 @@
 package org.fao.geonet.resources;
 
 import com.google.common.collect.Maps;
-
+import com.google.common.collect.Sets;
 import jeeves.config.springutil.JeevesDelegatingFilterProxy;
-
 import org.fao.geonet.NodeInfo;
-import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.domain.Pair;
 import org.fao.geonet.kernel.GeonetworkDataDirectory;
 import org.fao.geonet.kernel.setting.SettingManager;
-import org.fao.geonet.utils.Log;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.http.MediaType;
 import org.springframework.web.context.request.ServletWebRequest;
 
 import javax.servlet.*;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.attribute.FileTime;
@@ -83,6 +79,7 @@ public class Instance {
         private final ServletResponse response;
 
         private final Path resourcesDir;
+        private final Path schemaPublicationDir;
         private final Path appPath;
         private final String nodeId;
         private final String siteId;
@@ -96,6 +93,7 @@ public Instance(ServletRequest request, ServletResponse response) throws IOExcep
             this.resources = applicationContext.getBean(Resources.class);
             this.siteId = applicationContext.getBean(SettingManager.class).getSiteId();
             this.resourcesDir = resources.locateResourcesDir(servletContext, applicationContext);
+            this.schemaPublicationDir = applicationContext.getBean(GeonetworkDataDirectory.class).getSchemaPublicationDir();
             if (defaultImage == null) {
                 defaultImage = resources.loadResource(resourcesDir, servletContext, appPath, "images/logos/" + siteId + ".png", new byte[0], -1);
             }
@@ -133,10 +131,8 @@ public void execute() throws IOException {
                     return;
                 }
 
-                // Resources are images for logos, XML for map config or XSD
-                String contentType =  "xml".equals(ext) || "xsd".equals(ext)
-                    ? MediaType.APPLICATION_XML_VALUE
-                    : "image/" + ext;
+                // Figure out the content type based on the extensions, defaulting to an image
+                String contentType = extensionToMediaType(ext);
 
                 httpServletResponse.setContentType(contentType);
                 httpServletResponse.addHeader("Cache-Control", "max-age=" + SIX_HOURS + ", public");
@@ -148,6 +144,14 @@ public void execute() throws IOException {
                     httpServletResponse.setContentLength(favicon.one().length);
                     httpServletResponse.addHeader("Cache-Control", "max-age=" + FIVE_DAYS + ", public");
                     response.getOutputStream().write(favicon.one());
+                } else if(filename.startsWith("/xml/schemas/")) {
+                    Pair<byte[], Long> loadedResource = resources.loadResource(schemaPublicationDir, servletContext, appPath, filename, null, -1);
+                    if(loadedResource.two() == -1) {
+                        httpServletResponse.sendError(HttpServletResponse.SC_NOT_FOUND);
+                    } else {
+                        httpServletResponse.setContentLength(loadedResource.one().length);
+                        response.getOutputStream().write(loadedResource.one());
+                    }
                 } else {
                     byte[] defaultData = null;
 
@@ -175,4 +179,16 @@ private synchronized void addFavIcon(String nodeId, Pair<byte[], Long> favicon)
             }
         }
     }
+
+    private String extensionToMediaType(String ext) {
+        final String contentType;
+        if(Sets.newHashSet("xml", "xsd", "sch", "dtd").contains(ext)) {
+            contentType = MediaType.APPLICATION_XML_VALUE;
+        } else if(ext.equals("txt")) {
+            contentType = MediaType.TEXT_PLAIN_VALUE;
+        } else {
+            contentType = "image/" + ext;
+        }
+        return contentType;
+    }
 }

core/src/test/java/org/fao/geonet/GeonetTestFixture.java

@@ -181,12 +181,13 @@ protected void addSourceUUID(GeonetworkDataDirectory dataDirectory) {
     protected SchemaManager initSchemaManager(Path webappDir, GeonetworkDataDirectory geonetworkDataDirectory) throws Exception {
         final Path schemaPluginsDir = geonetworkDataDirectory.getSchemaPluginsDir();
         final Path resourcePath = geonetworkDataDirectory.getResourcesDir();
+        final Path schemaPublicationDir = geonetworkDataDirectory.getSchemaPublicationDir();
         Path schemaPluginsCatalogFile = schemaPluginsDir.resolve("schemaplugin-uri-catalog.xml");
 
         final SchemaManager schemaManager = _applicationContext.getBean(SchemaManager.class);
 
         SchemaManager.registerXmlCatalogFiles(webappDir, schemaPluginsCatalogFile);
-        schemaManager.configure(_applicationContext, webappDir, resourcePath,
+        schemaManager.configure(_applicationContext, webappDir, resourcePath, schemaPublicationDir,
             schemaPluginsCatalogFile, schemaPluginsDir, "eng", "iso19139", true);
 
         assertRequiredSchemas(schemaManager);

core/src/test/java/org/fao/geonet/kernel/AbstractGeonetworkDataDirectoryTest.java

@@ -64,6 +64,7 @@ public void testInit() throws Exception {
         dataDirectory.setHtmlCacheDir(null);
         dataDirectory.setSchemaPluginsDir(null);
         dataDirectory.setThesauriDir(null);
+        dataDirectory.setSchemaPublicationDir(null);
         final ArrayList<Element> serviceConfigParameterElements = getServiceConfigParameterElements();
         final ServiceConfig handlerConfig = new ServiceConfig(serviceConfigParameterElements);
         final Path webappDir = getWebappDir(getClass());
@@ -83,6 +84,7 @@ private void assertSystemDirSubFolders(Path expectedDataDir) {
         final Path expectedResourcesDir = expectedDataDir.resolve("data").resolve("resources");
         assertEquals(expectedResourcesDir, dataDirectory.getResourcesDir());
         assertEquals(expectedResourcesDir.resolve("htmlcache"), dataDirectory.getHtmlCacheDir());
+        assertEquals(expectedResourcesDir.resolve("schemapublication"), dataDirectory.getSchemaPublicationDir());
         assertEquals(expectedConfigDir.resolve("schema_plugins"), dataDirectory.getSchemaPluginsDir());
         assertEquals(expectedConfigDir.resolve("codelist"), dataDirectory.getThesauriDir());
     }

harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManager.java

@@ -30,7 +30,6 @@
 import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester;
 import org.jdom.Element;
 import org.quartz.SchedulerException;
-import org.springframework.context.ApplicationContext;
 import org.springframework.context.ConfigurableApplicationContext;
 
 import java.sql.SQLException;
@@ -52,6 +51,18 @@ public interface HarvestManager {
      */
     void init(@Nonnull ServiceContext context, boolean isReadOnly) throws Exception;
 
+    /**
+     * Refresh the harvesters.
+     * - Stops the existing scheduled jobs.
+     * - Reloads the harvester definitions from the database.
+     * - Schedule the jobs again.
+     * <p>
+     * Useful when running multiple instances as one instance is not aware of new harvesters added on the other one.
+     * <p>
+     * TODO when a durable solution for running highly available is implemented this scheduled task could be deleted, e.g., when created harvesters are notified through a shared message queue.
+     */
+    void refreshHarvesters();
+
     /**
      * Shutdown all harvesters and thread pools and jobs.
      */