Merge pull request #844 from lonvia/decouple-db-version

Decouple database versions for ES and OS implementation

app/es_embedded/src/main/java/de/komoot/photon/Server.java

@@ -37,6 +37,17 @@
  * Helper class to start/stop ElasticSearch node and get ElasticSearch clients.
  */
 public class Server {
+    /**
+     * Database version created by new imports with the current code.
+     *
+     * Format must be: major.minor.patch-dev
+     *
+     * Increase to next to be released version when the database layout
+     * changes in an incompatible way. If it is already at the next released
+     * version, increase the dev version.
+     */
+    public static final String DATABASE_VERSION = "0.3.6-1";
+
     private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class);
 
     public static final String PROPERTY_DOCUMENT_ID = "DATABASE_PROPERTIES";
@@ -173,9 +184,7 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate, boo
 
         createAndPutIndexMapping(languages, supportStructuredQueries);
 
-        DatabaseProperties dbProperties = new DatabaseProperties()
-            .setLanguages(languages)
-            .setImportDate(importDate);
+        DatabaseProperties dbProperties = new DatabaseProperties(languages, importDate, false);
         saveToDatabase(dbProperties);
 
         return dbProperties;
@@ -195,8 +204,7 @@ public void updateIndexSettings(String synonymFile) throws IOException {
         // Load the settings from the database to make sure it is at the right
         // version. If the version is wrong, we should not be messing with the
         // index.
-        DatabaseProperties dbProperties = new DatabaseProperties();
-        loadFromDatabase(dbProperties);
+        DatabaseProperties dbProperties = loadFromDatabase();
 
         loadIndexSettings().setSynonymFile(synonymFile).updateIndex(esClient, PhotonIndex.NAME);
 
@@ -228,7 +236,7 @@ private void deleteIndex() {
      */
     public void saveToDatabase(DatabaseProperties dbProperties) throws IOException  {
         final XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject(BASE_FIELD)
-                        .field(FIELD_VERSION, DatabaseProperties.DATABASE_VERSION)
+                        .field(FIELD_VERSION, DATABASE_VERSION)
                         .field(FIELD_LANGUAGES, String.join(",", dbProperties.getLanguages()))
                         .field(FIELD_IMPORT_DATE, dbProperties.getImportDate() instanceof Date ? dbProperties.getImportDate().toInstant() : null)
                         .endObject().endObject();
@@ -246,12 +254,12 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException
      * Currently does nothing when the property entry is missing. Later versions with a higher
      * database version will then fail.
      */
-    public void loadFromDatabase(DatabaseProperties dbProperties) {
+    public DatabaseProperties loadFromDatabase() {
         GetResponse response = esClient.prepareGet(PhotonIndex.NAME, PhotonIndex.TYPE, PROPERTY_DOCUMENT_ID).execute().actionGet();
 
         // We are currently at the database version where versioning was introduced.
         if (!response.isExists()) {
-            return;
+            throw new UsageException("Cannot find database properties. Your database version is too old. Please reimport.");
         }
 
         Map<String, String> properties = (Map<String, String>) response.getSource().get(BASE_FIELD);
@@ -261,16 +269,18 @@ public void loadFromDatabase(DatabaseProperties dbProperties) {
         }
 
         String version = properties.getOrDefault(FIELD_VERSION, "");
-        if (!DatabaseProperties.DATABASE_VERSION.equals(version)) {
-            LOGGER.error("Database has incompatible version '{}'. Expected: {}", version, DatabaseProperties.DATABASE_VERSION);
+        if (!DATABASE_VERSION.equals(version)) {
+            LOGGER.error("Database has incompatible version '{}'. Expected: {}",
+                         version, DATABASE_VERSION);
             throw new UsageException("Incompatible database.");
         }
 
         String langString = properties.get(FIELD_LANGUAGES);
-        dbProperties.setLanguages(langString == null ? null : langString.split(","));
-
         String importDateString = properties.get(FIELD_IMPORT_DATE);
-        dbProperties.setImportDate(importDateString == null ? null : Date.from(Instant.parse(importDateString)));
+
+        return new DatabaseProperties(langString == null ? null : langString.split(","),
+                                      importDateString == null ? null : Date.from(Instant.parse(importDateString)),
+                                      false);
     }
 
     public Importer createImporter(String[] languages, String[] extraTags) {

app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java

@@ -15,14 +15,11 @@ class ServerTest extends ESBaseTester {
     void testSaveAndLoadFromDatabase() throws IOException {
         setUpES();
 
-        DatabaseProperties prop = new DatabaseProperties();
-        prop.setLanguages(new String[]{"en", "de", "fr"});
         Date now = new Date();
-        prop.setImportDate(now);
+        DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false);
         getServer().saveToDatabase(prop);
 
-        prop = new DatabaseProperties();
-        getServer().loadFromDatabase(prop);
+        prop = getServer().loadFromDatabase();
 
         assertArrayEquals(new String[]{"en", "de", "fr"}, prop.getLanguages());
         assertEquals(now, prop.getImportDate());

app/opensearch/src/main/java/de/komoot/photon/Server.java

@@ -19,6 +19,17 @@
 import java.util.Date;
 
 public class Server {
+    /**
+     * Database version created by new imports with the current code.
+     *
+     * Format must be: major.minor.patch-dev
+     *
+     * Increase to next to be released version when the database layout
+     * changes in an incompatible way. If it is already at the next released
+     * version, increase the dev version.
+     */
+    public static final String DATABASE_VERSION = "0.3.6-1";
+
     private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class);
 
     protected OpenSearchClient client;
@@ -108,18 +119,14 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate, boo
 
         (new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME);
 
-        var dbProperties = new DatabaseProperties()
-                .setLanguages(languages)
-                .setSupportStructuredQueries(supportStructuredQueries)
-                .setImportDate(importDate);
+        var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries);
         saveToDatabase(dbProperties);
 
         return dbProperties;
     }
 
     public void updateIndexSettings(String synonymFile) throws IOException {
-        var dbProperties = new DatabaseProperties();
-        loadFromDatabase(dbProperties);
+        var dbProperties = loadFromDatabase();
 
         (new IndexSettingBuilder()).setSynonymFile(synonymFile).updateIndex(client, PhotonIndex.NAME);
 
@@ -134,11 +141,11 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException {
         client.index(r -> r
                         .index(PhotonIndex.NAME)
                         .id(PhotonIndex.PROPERTY_DOCUMENT_ID)
-                        .document(new DBPropertyEntry(dbProperties))
+                        .document(new DBPropertyEntry(dbProperties, DATABASE_VERSION))
                         );
     }
 
-    public void loadFromDatabase(DatabaseProperties dbProperties) throws IOException {
+    public DatabaseProperties loadFromDatabase() throws IOException {
         var dbEntry = client.get(r -> r
                 .index(PhotonIndex.NAME)
                 .id(PhotonIndex.PROPERTY_DOCUMENT_ID),
@@ -148,15 +155,15 @@ public void loadFromDatabase(DatabaseProperties dbProperties) throws IOException
             throw new UsageException("Cannot access property record. Database too old?");
         }
 
-        if (!DatabaseProperties.DATABASE_VERSION.equals(dbEntry.source().databaseVersion)) {
+        if (!DATABASE_VERSION.equals(dbEntry.source().databaseVersion)) {
             LOGGER.error("Database has incompatible version '{}'. Expected: {}",
-                         dbEntry.source().databaseVersion, DatabaseProperties.DATABASE_VERSION);
+                         dbEntry.source().databaseVersion, DATABASE_VERSION);
             throw new UsageException("Incompatible database.");
         }
 
-        dbProperties.setLanguages(dbEntry.source().languages);
-        dbProperties.setImportDate(dbEntry.source().importDate);
-        dbProperties.setSupportStructuredQueries(dbEntry.source().supportStructuredQueries);
+        return new DatabaseProperties(dbEntry.source().languages,
+                                      dbEntry.source().importDate,
+                                      dbEntry.source().supportStructuredQueries);
     }
 
     public Importer createImporter(String[] languages, String[] extraTags) {

app/opensearch/src/main/java/de/komoot/photon/opensearch/DBPropertyEntry.java

@@ -12,8 +12,8 @@ public class DBPropertyEntry {
 
     public DBPropertyEntry() {}
 
-    public DBPropertyEntry(DatabaseProperties props) {
-        databaseVersion = DatabaseProperties.DATABASE_VERSION;
+    public DBPropertyEntry(DatabaseProperties props, String databaseVersion) {
+        this.databaseVersion = databaseVersion;
         importDate = props.getImportDate();
         languages = props.getLanguages();
         supportStructuredQueries = props.getSupportStructuredQueries();

app/opensearch/src/test/java/de/komoot/photon/ServerTest.java

@@ -13,14 +13,13 @@ class ServerTest extends ESBaseTester {
     void testSaveAndLoadFromDatabase() throws IOException {
         setUpES();
 
-        DatabaseProperties prop = new DatabaseProperties();
-        prop.setLanguages(new String[]{"en", "de", "fr"});
         Date now = new Date();
-        prop.setImportDate(now);
+        DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"},
+                                                         now,
+                                   false);
         getServer().saveToDatabase(prop);
 
-        prop = new DatabaseProperties();
-        getServer().loadFromDatabase(prop);
+        prop = getServer().loadFromDatabase();
 
         assertArrayEquals(new String[]{"en", "de", "fr"}, prop.getLanguages());
         assertEquals(now, prop.getImportDate());

src/main/java/de/komoot/photon/App.java

@@ -145,9 +145,8 @@ private static void startNominatimUpdateInit(CommandLineArgs args) {
     private static void startNominatimUpdate(NominatimUpdater nominatimUpdater, Server esServer)  {
         nominatimUpdater.update();
 
-        DatabaseProperties dbProperties = new DatabaseProperties();
         try {
-            esServer.loadFromDatabase(dbProperties);
+            DatabaseProperties dbProperties = esServer.loadFromDatabase();
             Date importDate = nominatimUpdater.getLastImportDate();
             dbProperties.setImportDate(importDate);
             esServer.saveToDatabase(dbProperties);
@@ -162,8 +161,7 @@ private static void startNominatimUpdate(NominatimUpdater nominatimUpdater, Serv
      */
     private static NominatimUpdater setupNominatimUpdater(CommandLineArgs args, Server server)throws IOException {
         // Get database properties and ensure that the version is compatible.
-        DatabaseProperties dbProperties = new DatabaseProperties();
-        server.loadFromDatabase(dbProperties);
+        DatabaseProperties dbProperties = server.loadFromDatabase();
 
         NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
         nominatimUpdater.setUpdater(server.createUpdater(dbProperties.getLanguages(), args.getExtraTags()));
@@ -175,8 +173,7 @@ private static NominatimUpdater setupNominatimUpdater(CommandLineArgs args, Serv
      */
     private static void startApi(CommandLineArgs args, Server server) throws IOException {
         // Get database properties and ensure that the version is compatible.
-        DatabaseProperties dbProperties = new DatabaseProperties();
-        server.loadFromDatabase(dbProperties);
+        DatabaseProperties dbProperties = server.loadFromDatabase();
         if (args.getLanguages(false).length > 0) {
             dbProperties.restrictLanguages(args.getLanguages());
         }

src/main/java/de/komoot/photon/DatabaseProperties.java

@@ -8,29 +8,21 @@
  * The server is responsible for making the data persistent in the Photon database.
  */
 public class DatabaseProperties {
-    /**
-     * Database version created by new imports with the current code.
-     *
-     * Format must be: major.minor.patch-dev
-     *
-     * Increase to next to be released version when the database layout
-     * changes in an incompatible way. If it is already at the next released
-     * version, increase the dev version.
-     */
-    public static final String DATABASE_VERSION = "0.3.6-1";
-
-    private String[] languages = null;
-
-    /**
-     * The OSM data date
-     */
+    private String[] languages;
     private Date importDate;
+    private final boolean supportStructuredQueries;
 
-    private boolean supportStructuredQueries;
+    public DatabaseProperties(String[] languages, Date importDate, boolean supportStructuredQueries) {
+        this.languages = languages;
+        this.importDate = importDate;
+        this.supportStructuredQueries = supportStructuredQueries;
+    }
 
     /**
      * Return the list of languages for which the database is configured.
-     * @return
+     * If no list was set, then the default is returned.
+     *
+     * @return List of supported languages.
      */
     public String[] getLanguages() {
         if (languages == null) {
@@ -40,18 +32,6 @@ public String[] getLanguages() {
         return languages;
     }
 
-    /**
-     * Replace the language list with the given list.
-     *
-     * @param languages Array of two-letter language codes.
-     *
-     * @return This object for function chaining.
-     */
-    public DatabaseProperties setLanguages(String[] languages) {
-        this.languages = languages;
-        return this;
-    }
-
     /**
      * Set language list to the intersection between the existing list and the given list.
      *
@@ -83,21 +63,16 @@ public void restrictLanguages(String[] languageList) {
         }
     }
 
-    public Date getImportDate() {
-        return this.importDate;
+     public void setImportDate(Date importDate) {
+        this.importDate = importDate;
     }
 
-    public DatabaseProperties setImportDate(Date importDate) {
-        this.importDate = importDate;
-        return this;
+
+    public Date getImportDate() {
+        return this.importDate;
     }
 
     public boolean getSupportStructuredQueries() {
         return supportStructuredQueries;
     }
-
-    public DatabaseProperties setSupportStructuredQueries(boolean supportStructuredQueries) {
-        this.supportStructuredQueries = supportStructuredQueries;
-        return this;
-    }
 }

src/main/java/de/komoot/photon/StatusRequestHandler.java

@@ -19,8 +19,7 @@ protected StatusRequestHandler(String path, Server server) {
 
     @Override
     public String handle(Request request, Response response) throws IOException {
-        DatabaseProperties dbProperties = new DatabaseProperties();
-        server.loadFromDatabase(dbProperties);
+        DatabaseProperties dbProperties = server.loadFromDatabase();
         String importDateStr = null;
         if (dbProperties.getImportDate() instanceof Date) {
             importDateStr = dbProperties.getImportDate().toInstant().toString();

src/test/java/de/komoot/photon/ApiIntegrationTest.java

@@ -139,8 +139,7 @@ void testApi(String osmValue, String url) throws Exception {
     void testApiStatus() throws Exception {
         App.main(new String[]{"-cluster", TEST_CLUSTER_NAME, "-listen-port", Integer.toString(LISTEN_PORT), "-transport-addresses", "127.0.0.1"});
         awaitInitialization();
-        DatabaseProperties prop = new DatabaseProperties();
-        getServer().loadFromDatabase(prop);
+        DatabaseProperties prop = getServer().loadFromDatabase();
         HttpURLConnection connection = (HttpURLConnection) new URL("http://127.0.0.1:" + port() + "/status").openConnection();
         JSONObject json = new JSONObject(
                 new BufferedReader(new InputStreamReader(connection.getInputStream())).lines().collect(Collectors.joining("\n")));

src/test/java/de/komoot/photon/DatabasePropertiesTest.java

@@ -16,24 +16,19 @@ class DatabasePropertiesTest extends ESBaseTester {
      */
     @Test
     void testSetLanguages() {
-        DatabaseProperties prop = new DatabaseProperties();
+        var now = new Date();
+        DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "bg", "de"}, now, false);
 
-        prop.setLanguages(new String[]{"en", "bg", "de"});
-        Date now = new Date();
-        prop.setImportDate(now);
         assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages());
         assertEquals(now, prop.getImportDate());
-
-        prop.setLanguages(new String[]{"ru"});
-        assertArrayEquals(new String[]{"ru"}, prop.getLanguages());
     }
 
     /**
      * If languages is not set, then the restricted language set is used as is.
      */
     @Test
     void testRestrictLanguagesUnsetLanguages() {
-        DatabaseProperties prop = new DatabaseProperties();
+        DatabaseProperties prop = new DatabaseProperties(null, null, false);
         prop.restrictLanguages(new String[]{"en", "bg", "de"});
 
         assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages());
@@ -45,8 +40,7 @@ void testRestrictLanguagesUnsetLanguages() {
      */
     @Test
     void testRestrictLanguagesAlreadySet() {
-        DatabaseProperties prop = new DatabaseProperties();
-        prop.setLanguages(new String[]{"en", "de", "fr"});
+        DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, null, false);
 
         prop.restrictLanguages(new String[]{"cn", "de", "en", "es"});