Merge pull request #635 from lonvia/new-interpolation-structure

Support for the changed schema of Nominatim's interpolation tables

src/main/java/de/komoot/photon/App.java

@@ -175,11 +175,13 @@ private static void startApi(CommandLineArgs args, Client esNodeClient) {
         get("reverse", new ReverseSearchRequestHandler("reverse", esNodeClient, dbProperties.getLanguages(), args.getDefaultLanguage()));
         get("reverse/", new ReverseSearchRequestHandler("reverse/", esNodeClient, dbProperties.getLanguages(), args.getDefaultLanguage()));
 
-        // setup update API
-        final NominatimUpdater nominatimUpdater = setupNominatimUpdater(args, esNodeClient);
-        get("/nominatim-update", (Request request, Response response) -> {
-            new Thread(() -> nominatimUpdater.update()).start();
-            return "nominatim update started (more information in console output) ...";
-        });
+        if (args.isEnableUpdateApi()) {
+            // setup update API
+            final NominatimUpdater nominatimUpdater = setupNominatimUpdater(args, esNodeClient);
+            get("/nominatim-update", (Request request, Response response) -> {
+                new Thread(() -> nominatimUpdater.update()).start();
+                return "nominatim update started (more information in console output) ...";
+            });
+        }
     }
 }

src/main/java/de/komoot/photon/CommandLineArgs.java

@@ -71,6 +71,9 @@ public class CommandLineArgs {
     @Parameter(names = "-cors-origin", description = "enable cross-site resource sharing for the specified origin (default CORS not supported)")
     private String corsOrigin = null;
 
+    @Parameter(names = "-enable-update-api", description = "Enable the additional endpoint /nominatim-update, which allows to trigger updates from a nominatim database")
+    private boolean enableUpdateApi = false;
+
     @Parameter(names = "-h", description = "show help / usage")
     private boolean usage = false;
 

src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java

@@ -1,6 +1,7 @@
 package de.komoot.photon.nominatim;
 
 import com.vividsolutions.jts.geom.Geometry;
+import org.springframework.jdbc.core.JdbcTemplate;
 
 import javax.annotation.Nullable;
 import java.sql.ResultSet;
@@ -21,4 +22,9 @@ public interface DBDataAdapter {
      */
     @Nullable
     Geometry extractGeometry(ResultSet rs, String columnName) throws SQLException;
+
+    /**
+     * Check if a table has the given column.
+     */
+    boolean hasColumn(JdbcTemplate template, String table, String column);
 }

src/main/java/de/komoot/photon/nominatim/NominatimConnector.java

@@ -26,37 +26,20 @@
 @Slf4j
 public class NominatimConnector {
     private static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid";
-    private static final String SELECT_COLS_OSMLINE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
     private static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address";
 
     private final DBDataAdapter dbutils;
     private final JdbcTemplate template;
     private Map<String, Map<String, String>> countryNames;
+
     /**
-     * Maps a row from location_property_osmline (address interpolation lines) to a photon doc.
+     * Maps a row from location_property_osmline (address interpolation lines)
+     * with old-style intepolation (using interpolationtype) to a photon doc.
      */
-    private final RowMapper<NominatimResult> osmlineRowMapper = new RowMapper<NominatimResult>() {
-        @Override
-        public NominatimResult mapRow(ResultSet rs, int rownum) throws SQLException {
-            Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
-
-            PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"),
-                                          "place", "house_number")
-                    .parentPlaceId(rs.getLong("parent_place_id"))
-                    .countryCode(rs.getString("country_code"))
-                    .postcode(rs.getString("postcode"));
-
-            completePlace(doc);
-
-            doc.setCountry(getCountryNames(rs.getString("country_code")));
+    private final RowMapper<NominatimResult> osmlineRowMapper;
+    private final String selectOsmlineSql;
 
-            NominatimResult result = new NominatimResult(doc);
-            result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"),
-                    rs.getString("interpolationtype"), geometry);
 
-            return result;
-        }
-    };
     /**
      * maps a placex row in nominatim to a photon doc, some attributes are still missing and can be derived by connected address items.
      */
@@ -112,6 +95,53 @@ public NominatimConnector(String host, int port, String database, String usernam
         template.setFetchSize(100000);
 
         dbutils = dataAdapter;
+
+        // Setup handling of interpolation table. It has changed its format. Need to find out which one to use.
+        if (dbutils.hasColumn(template, "location_property_osmline", "step")) {
+            // new-style interpolations
+            selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo";
+            osmlineRowMapper = (rs, rownum) -> {
+                Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
+
+                PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"),
+                        "place", "house_number")
+                        .parentPlaceId(rs.getLong("parent_place_id"))
+                        .countryCode(rs.getString("country_code"))
+                        .postcode(rs.getString("postcode"));
+
+                completePlace(doc);
+
+                doc.setCountry(getCountryNames(rs.getString("country_code")));
+
+                NominatimResult result = new NominatimResult(doc);
+                result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"),
+                        rs.getLong("step"), geometry);
+
+                return result;
+            };
+        } else {
+            // old-style interpolations
+            selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
+            osmlineRowMapper = (rs, rownum) -> {
+                Geometry geometry = dbutils.extractGeometry(rs, "linegeo");
+
+                PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"),
+                        "place", "house_number")
+                        .parentPlaceId(rs.getLong("parent_place_id"))
+                        .countryCode(rs.getString("country_code"))
+                        .postcode(rs.getString("postcode"));
+
+                completePlace(doc);
+
+                doc.setCountry(getCountryNames(rs.getString("country_code")));
+
+                NominatimResult result = new NominatimResult(doc);
+                result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"),
+                        rs.getString("interpolationtype"), geometry);
+
+                return result;
+            };
+        }
     }
 
 
@@ -150,7 +180,7 @@ public List<PhotonDoc> getByPlaceId(long placeId) {
     }
 
     public List<PhotonDoc> getInterpolationsByPlaceId(long placeId) {
-        NominatimResult result = template.queryForObject(SELECT_COLS_OSMLINE
+        NominatimResult result = template.queryForObject(selectOsmlineSql
                                                           + " FROM location_property_osmline WHERE place_id = ?",
                                                           osmlineRowMapper, placeId);
         assert(result != null);
@@ -227,11 +257,9 @@ static String convertCountryCode(String... countryCodes) {
      */
     public void readEntireDatabase(String... countryCodes) {
         String andCountryCodeStr = "";
-        String whereCountryCodeStr = "";
         String countryCodeStr = convertCountryCode(countryCodes);
         if (!countryCodeStr.isEmpty()) {
             andCountryCodeStr = "AND country_code in (" + countryCodeStr + ")";
-            whereCountryCodeStr = "WHERE country_code in (" + countryCodeStr + ")";
         }
 
         log.info("start importing documents from nominatim (" + (countryCodeStr.isEmpty() ? "global" : countryCodeStr) + ")");
@@ -250,8 +278,9 @@ public void readEntireDatabase(String... countryCodes) {
                     }
                 });
 
-        template.query(SELECT_COLS_OSMLINE + " FROM location_property_osmline " +
-                whereCountryCodeStr +
+        template.query(selectOsmlineSql + " FROM location_property_osmline " +
+                "WHERE startnumber is not null " +
+                andCountryCodeStr +
                 " ORDER BY geometry_sector, parent_place_id; ", rs -> {
                     NominatimResult docs = osmlineRowMapper.mapRow(rs, 0);
                     assert(docs != null);

src/main/java/de/komoot/photon/nominatim/NominatimResult.java

@@ -83,6 +83,18 @@ public void addHousenumbersFromAddress(Map<String, String> address) {
         addHousenumbersFromString(address.get("conscriptionnumber"));
     }
 
+    /**
+     * Add old-style interpolated housenumbers.
+     *
+     * Old-style interpolation include the start and end point of the interpolation which is normally also
+     * an OSM house number object. They also feature only an interpolation type (odd, even, all) which may
+     * require some correction of the start value.
+     *
+     * @param first First number in the interpolation.
+     * @param last Last number in the interpolation.
+     * @param interpoltype Kind of interpolation (odd, even or all).
+     * @param geom Geometry of the interpolation line.
+     */
     public void addHouseNumbersFromInterpolation(long first, long last, String interpoltype, Geometry geom) {
         if (last <= first || (last - first) > 1000)
             return;
@@ -113,4 +125,33 @@ public void addHouseNumbersFromInterpolation(long first, long last, String inter
             housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num)));
         }
     }
+
+    /**
+     * Add new-style interpolated house numbers.
+     *
+     * New-style interpolations only have a step with and first and last are included in the numbers that
+     * need interpolation.
+     *
+     * @param first First number of the interpolation.
+     * @param last Last number of the interpolation.
+     * @param step Gap to leave between each interpolated housenumber.
+     * @param geom Geometry of the interpolation line.
+     */
+    public void addHouseNumbersFromInterpolation(long first, long last, long step, Geometry geom) {
+         if (last <= first || (last - first) > 1000)
+            return;
+
+        if (housenumbers == null)
+            housenumbers = new HashMap<>();
+
+        LengthIndexedLine line = new LengthIndexedLine(geom);
+        double si = line.getStartIndex();
+        double ei = line.getEndIndex();
+        double lstep = (ei - si) / (double) (last - first);
+
+        GeometryFactory fac = geom.getFactory();
+        for (long num = 1; first + num <= last; num += step) {
+            housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num)));
+        }
+    }
 }

src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java

@@ -3,6 +3,8 @@
 import com.google.common.collect.Maps;
 import com.vividsolutions.jts.geom.Geometry;
 import org.postgis.jts.JtsGeometry;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.RowMapper;
 
 import javax.annotation.Nullable;
 import java.sql.ResultSet;
@@ -35,4 +37,15 @@ public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLExcep
         }
         return geom.getGeometry();
     }
+
+    @Override
+    public boolean hasColumn(JdbcTemplate template, String table, String column) {
+        return template.query("SELECT count(*) FROM information_schema.columns WHERE table_name = ? and column_name = ?",
+                new RowMapper<Boolean>() {
+                    @Override
+                    public Boolean mapRow(ResultSet resultSet, int i) throws SQLException {
+                        return resultSet.getInt(1) > 0;
+                    }
+                }, table, column).get(0);
+    }
 }

src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java

@@ -5,6 +5,7 @@
 import com.vividsolutions.jts.io.WKTReader;
 import de.komoot.photon.nominatim.DBDataAdapter;
 import org.json.JSONObject;
+import org.springframework.jdbc.core.JdbcTemplate;
 
 import javax.annotation.Nullable;
 import java.sql.ResultSet;
@@ -42,4 +43,9 @@ public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLExcep
 
         return null;
     }
+
+    @Override
+    public boolean hasColumn(JdbcTemplate template, String table, String column) {
+        return false;
+    }
 }