diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java index 33868a6c3..321391cf1 100644 --- a/src/main/java/de/komoot/photon/App.java +++ b/src/main/java/de/komoot/photon/App.java @@ -175,11 +175,13 @@ private static void startApi(CommandLineArgs args, Client esNodeClient) { get("reverse", new ReverseSearchRequestHandler("reverse", esNodeClient, dbProperties.getLanguages(), args.getDefaultLanguage())); get("reverse/", new ReverseSearchRequestHandler("reverse/", esNodeClient, dbProperties.getLanguages(), args.getDefaultLanguage())); - // setup update API - final NominatimUpdater nominatimUpdater = setupNominatimUpdater(args, esNodeClient); - get("/nominatim-update", (Request request, Response response) -> { - new Thread(() -> nominatimUpdater.update()).start(); - return "nominatim update started (more information in console output) ..."; - }); + if (args.isEnableUpdateApi()) { + // setup update API + final NominatimUpdater nominatimUpdater = setupNominatimUpdater(args, esNodeClient); + get("/nominatim-update", (Request request, Response response) -> { + new Thread(() -> nominatimUpdater.update()).start(); + return "nominatim update started (more information in console output) ..."; + }); + } } } diff --git a/src/main/java/de/komoot/photon/CommandLineArgs.java b/src/main/java/de/komoot/photon/CommandLineArgs.java index 2c7eb51be..f2d6591c6 100644 --- a/src/main/java/de/komoot/photon/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/CommandLineArgs.java @@ -71,6 +71,9 @@ public class CommandLineArgs { @Parameter(names = "-cors-origin", description = "enable cross-site resource sharing for the specified origin (default CORS not supported)") private String corsOrigin = null; + @Parameter(names = "-enable-update-api", description = "Enable the additional endpoint /nominatim-update, which allows to trigger updates from a nominatim database") + private boolean enableUpdateApi = false; + @Parameter(names = "-h", description = "show help / usage") private boolean usage = false; diff --git a/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java b/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java index 422070257..016b2a2b7 100644 --- a/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java +++ b/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java @@ -1,6 +1,7 @@ package de.komoot.photon.nominatim; import com.vividsolutions.jts.geom.Geometry; +import org.springframework.jdbc.core.JdbcTemplate; import javax.annotation.Nullable; import java.sql.ResultSet; @@ -21,4 +22,9 @@ public interface DBDataAdapter { */ @Nullable Geometry extractGeometry(ResultSet rs, String columnName) throws SQLException; + + /** + * Check if a table has the given column. + */ + boolean hasColumn(JdbcTemplate template, String table, String column); } diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java index 05a3ccc6b..fb18e8c44 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java @@ -26,37 +26,20 @@ @Slf4j public class NominatimConnector { private static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid"; - private static final String SELECT_COLS_OSMLINE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo"; private static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address"; private final DBDataAdapter dbutils; private final JdbcTemplate template; private Map> countryNames; + /** - * Maps a row from location_property_osmline (address interpolation lines) to a photon doc. + * Maps a row from location_property_osmline (address interpolation lines) + * with old-style intepolation (using interpolationtype) to a photon doc. */ - private final RowMapper osmlineRowMapper = new RowMapper() { - @Override - public NominatimResult mapRow(ResultSet rs, int rownum) throws SQLException { - Geometry geometry = dbutils.extractGeometry(rs, "linegeo"); - - PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"), - "place", "house_number") - .parentPlaceId(rs.getLong("parent_place_id")) - .countryCode(rs.getString("country_code")) - .postcode(rs.getString("postcode")); - - completePlace(doc); - - doc.setCountry(getCountryNames(rs.getString("country_code"))); + private final RowMapper osmlineRowMapper; + private final String selectOsmlineSql; - NominatimResult result = new NominatimResult(doc); - result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"), - rs.getString("interpolationtype"), geometry); - return result; - } - }; /** * maps a placex row in nominatim to a photon doc, some attributes are still missing and can be derived by connected address items. */ @@ -112,6 +95,53 @@ public NominatimConnector(String host, int port, String database, String usernam template.setFetchSize(100000); dbutils = dataAdapter; + + // Setup handling of interpolation table. It has changed its format. Need to find out which one to use. + if (dbutils.hasColumn(template, "location_property_osmline", "step")) { + // new-style interpolations + selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo"; + osmlineRowMapper = (rs, rownum) -> { + Geometry geometry = dbutils.extractGeometry(rs, "linegeo"); + + PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"), + "place", "house_number") + .parentPlaceId(rs.getLong("parent_place_id")) + .countryCode(rs.getString("country_code")) + .postcode(rs.getString("postcode")); + + completePlace(doc); + + doc.setCountry(getCountryNames(rs.getString("country_code"))); + + NominatimResult result = new NominatimResult(doc); + result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"), + rs.getLong("step"), geometry); + + return result; + }; + } else { + // old-style interpolations + selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo"; + osmlineRowMapper = (rs, rownum) -> { + Geometry geometry = dbutils.extractGeometry(rs, "linegeo"); + + PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"), + "place", "house_number") + .parentPlaceId(rs.getLong("parent_place_id")) + .countryCode(rs.getString("country_code")) + .postcode(rs.getString("postcode")); + + completePlace(doc); + + doc.setCountry(getCountryNames(rs.getString("country_code"))); + + NominatimResult result = new NominatimResult(doc); + result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"), + rs.getString("interpolationtype"), geometry); + + return result; + }; + } } @@ -150,7 +180,7 @@ public List getByPlaceId(long placeId) { } public List getInterpolationsByPlaceId(long placeId) { - NominatimResult result = template.queryForObject(SELECT_COLS_OSMLINE + NominatimResult result = template.queryForObject(selectOsmlineSql + " FROM location_property_osmline WHERE place_id = ?", osmlineRowMapper, placeId); assert(result != null); @@ -227,11 +257,9 @@ static String convertCountryCode(String... countryCodes) { */ public void readEntireDatabase(String... countryCodes) { String andCountryCodeStr = ""; - String whereCountryCodeStr = ""; String countryCodeStr = convertCountryCode(countryCodes); if (!countryCodeStr.isEmpty()) { andCountryCodeStr = "AND country_code in (" + countryCodeStr + ")"; - whereCountryCodeStr = "WHERE country_code in (" + countryCodeStr + ")"; } log.info("start importing documents from nominatim (" + (countryCodeStr.isEmpty() ? "global" : countryCodeStr) + ")"); @@ -250,8 +278,9 @@ public void readEntireDatabase(String... countryCodes) { } }); - template.query(SELECT_COLS_OSMLINE + " FROM location_property_osmline " + - whereCountryCodeStr + + template.query(selectOsmlineSql + " FROM location_property_osmline " + + "WHERE startnumber is not null " + + andCountryCodeStr + " ORDER BY geometry_sector, parent_place_id; ", rs -> { NominatimResult docs = osmlineRowMapper.mapRow(rs, 0); assert(docs != null); diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java index e6b1b2d36..9285b10aa 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java @@ -83,6 +83,18 @@ public void addHousenumbersFromAddress(Map address) { addHousenumbersFromString(address.get("conscriptionnumber")); } + /** + * Add old-style interpolated housenumbers. + * + * Old-style interpolation include the start and end point of the interpolation which is normally also + * an OSM house number object. They also feature only an interpolation type (odd, even, all) which may + * require some correction of the start value. + * + * @param first First number in the interpolation. + * @param last Last number in the interpolation. + * @param interpoltype Kind of interpolation (odd, even or all). + * @param geom Geometry of the interpolation line. + */ public void addHouseNumbersFromInterpolation(long first, long last, String interpoltype, Geometry geom) { if (last <= first || (last - first) > 1000) return; @@ -113,4 +125,33 @@ public void addHouseNumbersFromInterpolation(long first, long last, String inter housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num))); } } + + /** + * Add new-style interpolated house numbers. + * + * New-style interpolations only have a step with and first and last are included in the numbers that + * need interpolation. + * + * @param first First number of the interpolation. + * @param last Last number of the interpolation. + * @param step Gap to leave between each interpolated housenumber. + * @param geom Geometry of the interpolation line. + */ + public void addHouseNumbersFromInterpolation(long first, long last, long step, Geometry geom) { + if (last <= first || (last - first) > 1000) + return; + + if (housenumbers == null) + housenumbers = new HashMap<>(); + + LengthIndexedLine line = new LengthIndexedLine(geom); + double si = line.getStartIndex(); + double ei = line.getEndIndex(); + double lstep = (ei - si) / (double) (last - first); + + GeometryFactory fac = geom.getFactory(); + for (long num = 1; first + num <= last; num += step) { + housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num))); + } + } } diff --git a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java index be05bcc3a..7eef7cdcc 100644 --- a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java +++ b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java @@ -3,6 +3,8 @@ import com.google.common.collect.Maps; import com.vividsolutions.jts.geom.Geometry; import org.postgis.jts.JtsGeometry; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; import javax.annotation.Nullable; import java.sql.ResultSet; @@ -35,4 +37,15 @@ public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLExcep } return geom.getGeometry(); } + + @Override + public boolean hasColumn(JdbcTemplate template, String table, String column) { + return template.query("SELECT count(*) FROM information_schema.columns WHERE table_name = ? and column_name = ?", + new RowMapper() { + @Override + public Boolean mapRow(ResultSet resultSet, int i) throws SQLException { + return resultSet.getInt(1) > 0; + } + }, table, column).get(0); + } } diff --git a/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java b/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java index 23667ec0d..b637ff85a 100644 --- a/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java +++ b/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java @@ -5,6 +5,7 @@ import com.vividsolutions.jts.io.WKTReader; import de.komoot.photon.nominatim.DBDataAdapter; import org.json.JSONObject; +import org.springframework.jdbc.core.JdbcTemplate; import javax.annotation.Nullable; import java.sql.ResultSet; @@ -42,4 +43,9 @@ public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLExcep return null; } + + @Override + public boolean hasColumn(JdbcTemplate template, String table, String column) { + return false; + } }