Merge pull request #3233 from lonvia/support-for-sqlite

Add support for SQLite DBs in frontend: reverse

.github/actions/build-nominatim/action.yml

@@ -25,12 +25,12 @@ runs:
           shell: bash
         - name: Install${{ matrix.flavour }} prerequisites
           run: |
-            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson nlohmann-json3-dev
+            sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua${LUA_VERSION} lua-dkjson nlohmann-json3-dev libspatialite7 libsqlite3-mod-spatialite
             if [ "$FLAVOUR" == "oldstuff" ]; then
-                pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4.31 datrie asyncpg
+                pip3 install MarkupSafe==2.0.1 python-dotenv psycopg2==2.7.7 jinja2==2.8 psutil==5.4.2 pyicu==2.9 osmium PyYAML==5.1 sqlalchemy==1.4.31 datrie asyncpg aiosqlite
             else
                 sudo apt-get install -y -qq python3-icu python3-datrie python3-pyosmium python3-jinja2 python3-psutil python3-psycopg2 python3-dotenv python3-yaml
-                pip3 install sqlalchemy psycopg
+                pip3 install sqlalchemy psycopg aiosqlite
             fi
           shell: bash
           env:

.github/workflows/ci-tests.yml

@@ -113,18 +113,21 @@ jobs:
               if: matrix.flavour == 'oldstuff'
 
             - name: Install Python webservers
-              run: pip3 install falcon starlette
+              run: pip3 install falcon starlette asgi_lifespan
 
             - name: Install latest pylint
-              run: pip3 install -U pylint asgi_lifespan
+              run: pip3 install -U pylint
+              if: matrix.flavour != 'oldstuff'
 
             - name: PHP linting
               run: phpcs --report-width=120 .
               working-directory: Nominatim
+              if: matrix.flavour != 'oldstuff'
 
             - name: Python linting
               run: python3 -m pylint nominatim
               working-directory: Nominatim
+              if: matrix.flavour != 'oldstuff'
 
             - name: PHP unit tests
               run: phpunit ./

nominatim/api/core.py

@@ -81,21 +81,34 @@ async def setup_database(self) -> None:
             if self._engine:
                 return
 
-            dsn = self.config.get_database_params()
-            pool_size = self.config.get_int('API_POOL_SIZE')
-
-            query = {k: v for k, v in dsn.items()
-                      if k not in ('user', 'password', 'dbname', 'host', 'port')}
-
-            dburl = sa.engine.URL.create(
-                       f'postgresql+{PGCORE_LIB}',
-                       database=dsn.get('dbname'),
-                       username=dsn.get('user'), password=dsn.get('password'),
-                       host=dsn.get('host'), port=int(dsn['port']) if 'port' in dsn else None,
-                       query=query)
-            engine = sa_asyncio.create_async_engine(dburl, future=True,
-                                                    max_overflow=0, pool_size=pool_size,
-                                                    echo=self.config.get_bool('DEBUG_SQL'))
+            extra_args: Dict[str, Any] = {'future': True,
+                                          'echo': self.config.get_bool('DEBUG_SQL')}
+
+            is_sqlite = self.config.DATABASE_DSN.startswith('sqlite:')
+
+            if is_sqlite:
+                params = dict((p.split('=', 1)
+                              for p in self.config.DATABASE_DSN[7:].split(';')))
+                dburl = sa.engine.URL.create('sqlite+aiosqlite',
+                                             database=params.get('dbname'))
+
+            else:
+                dsn = self.config.get_database_params()
+                query = {k: v for k, v in dsn.items()
+                         if k not in ('user', 'password', 'dbname', 'host', 'port')}
+
+                dburl = sa.engine.URL.create(
+                           f'postgresql+{PGCORE_LIB}',
+                           database=dsn.get('dbname'),
+                           username=dsn.get('user'),
+                           password=dsn.get('password'),
+                           host=dsn.get('host'),
+                           port=int(dsn['port']) if 'port' in dsn else None,
+                           query=query)
+                extra_args['max_overflow'] = 0
+                extra_args['pool_size'] = self.config.get_int('API_POOL_SIZE')
+
+            engine = sa_asyncio.create_async_engine(dburl, **extra_args)
 
             try:
                 async with engine.begin() as conn:
@@ -104,7 +117,7 @@ async def setup_database(self) -> None:
             except (PGCORE_ERROR, sa.exc.OperationalError):
                 server_version = 0
 
-            if server_version >= 110000:
+            if server_version >= 110000 and not is_sqlite:
                 @sa.event.listens_for(engine.sync_engine, "connect")
                 def _on_connect(dbapi_con: Any, _: Any) -> None:
                     cursor = dbapi_con.cursor()
@@ -113,6 +126,15 @@ def _on_connect(dbapi_con: Any, _: Any) -> None:
                 # Make sure that all connections get the new settings
                 await self.close()
 
+            if is_sqlite:
+                @sa.event.listens_for(engine.sync_engine, "connect")
+                def _on_sqlite_connect(dbapi_con: Any, _: Any) -> None:
+                    dbapi_con.run_async(lambda conn: conn.enable_load_extension(True))
+                    cursor = dbapi_con.cursor()
+                    cursor.execute("SELECT load_extension('mod_spatialite')")
+                    cursor.execute('SELECT SetDecimalPrecision(7)')
+                    dbapi_con.run_async(lambda conn: conn.enable_load_extension(False))
+
             self._property_cache['DB:server_version'] = server_version
 
             self._tables = SearchTables(sa.MetaData(), engine.name) # pylint: disable=no-member

nominatim/api/lookup.py

@@ -77,8 +77,8 @@ async def find_in_osmline(conn: SearchConnection, place: ntyp.PlaceRef,
         sql = sql.where(t.c.osm_id == place.osm_id).limit(1)
         if place.osm_class and place.osm_class.isdigit():
             sql = sql.order_by(sa.func.greatest(0,
-                                    sa.func.least(int(place.osm_class) - t.c.endnumber),
-                                           t.c.startnumber - int(place.osm_class)))
+                                                int(place.osm_class) - t.c.endnumber,
+                                                t.c.startnumber - int(place.osm_class)))
     else:
         return None
 
@@ -163,11 +163,10 @@ async def get_detailed_place(conn: SearchConnection, place: ntyp.PlaceRef,
 
     if details.geometry_output & ntyp.GeometryFormat.GEOJSON:
         def _add_geometry(sql: SaSelect, column: SaColumn) -> SaSelect:
-            return sql.add_columns(sa.literal_column(f"""
-                      ST_AsGeoJSON(CASE WHEN ST_NPoints({column.name}) > 5000
-                                   THEN ST_SimplifyPreserveTopology({column.name}, 0.0001)
-                                   ELSE {column.name} END)
-                       """).label('geometry_geojson'))
+            return sql.add_columns(sa.func.ST_AsGeoJSON(
+                                    sa.case((sa.func.ST_NPoints(column) > 5000,
+                                             sa.func.ST_SimplifyPreserveTopology(column, 0.0001)),
+                                            else_=column), 7).label('geometry_geojson'))
     else:
         def _add_geometry(sql: SaSelect, column: SaColumn) -> SaSelect:
             return sql.add_columns(sa.func.ST_GeometryType(column).label('geometry_type'))
@@ -183,6 +182,9 @@ def _add_geometry(sql: SaSelect, column: SaColumn) -> SaSelect:
 
     # add missing details
     assert result is not None
+    if 'type' in result.geometry:
+        result.geometry['type'] = GEOMETRY_TYPE_MAP.get(result.geometry['type'],
+                                                        result.geometry['type'])
     indexed_date = getattr(row, 'indexed_date', None)
     if indexed_date is not None:
         result.indexed_date = indexed_date.replace(tzinfo=dt.timezone.utc)
@@ -208,13 +210,13 @@ def _add_geometry(sql: SaSelect, col: SaColumn) -> SaSelect:
             col = sa.func.ST_SimplifyPreserveTopology(col, details.geometry_simplification)
 
         if details.geometry_output & ntyp.GeometryFormat.GEOJSON:
-            out.append(sa.func.ST_AsGeoJSON(col).label('geometry_geojson'))
+            out.append(sa.func.ST_AsGeoJSON(col, 7).label('geometry_geojson'))
         if details.geometry_output & ntyp.GeometryFormat.TEXT:
             out.append(sa.func.ST_AsText(col).label('geometry_text'))
         if details.geometry_output & ntyp.GeometryFormat.KML:
-            out.append(sa.func.ST_AsKML(col).label('geometry_kml'))
+            out.append(sa.func.ST_AsKML(col, 7).label('geometry_kml'))
         if details.geometry_output & ntyp.GeometryFormat.SVG:
-            out.append(sa.func.ST_AsSVG(col).label('geometry_svg'))
+            out.append(sa.func.ST_AsSVG(col, 0, 7).label('geometry_svg'))
 
         return sql.add_columns(*out)
 
@@ -236,3 +238,14 @@ def _add_geometry(sql: SaSelect, col: SaColumn) -> SaSelect:
     await nres.add_result_details(conn, [result], details)
 
     return result
+
+
+GEOMETRY_TYPE_MAP = {
+    'POINT': 'ST_Point',
+    'MULTIPOINT': 'ST_MultiPoint',
+    'LINESTRING': 'ST_LineString',
+    'MULTILINESTRING': 'ST_MultiLineString',
+    'POLYGON': 'ST_Polygon',
+    'MULTIPOLYGON': 'ST_MultiPolygon',
+    'GEOMETRYCOLLECTION': 'ST_GeometryCollection'
+}

nominatim/api/results.py

@@ -19,7 +19,7 @@
 import sqlalchemy as sa
 
 from nominatim.typing import SaSelect, SaRow
-from nominatim.db.sqlalchemy_functions import CrosscheckNames
+from nominatim.db.sqlalchemy_types import Geometry
 from nominatim.api.types import Point, Bbox, LookupDetails
 from nominatim.api.connection import SearchConnection
 from nominatim.api.logging import log
@@ -589,7 +589,7 @@ async def complete_address_details(conn: SearchConnection, results: List[BaseRes
     if not lookup_ids:
         return
 
-    ltab = sa.func.json_array_elements(sa.type_coerce(lookup_ids, sa.JSON))\
+    ltab = sa.func.JsonArrayEach(sa.type_coerce(lookup_ids, sa.JSON))\
              .table_valued(sa.column('value', type_=sa.JSON)) # type: ignore[no-untyped-call]
 
     t = conn.t.placex
@@ -608,7 +608,7 @@ async def complete_address_details(conn: SearchConnection, results: List[BaseRes
             .order_by('src_place_id')\
             .order_by(sa.column('rank_address').desc())\
             .order_by((taddr.c.place_id == ltab.c.value['pid'].as_integer()).desc())\
-            .order_by(sa.case((CrosscheckNames(t.c.name, ltab.c.value['names']), 2),
+            .order_by(sa.case((sa.func.CrosscheckNames(t.c.name, ltab.c.value['names']), 2),
                               (taddr.c.isaddress, 0),
                               (sa.and_(taddr.c.fromarea,
                                        t.c.geometry.ST_Contains(
@@ -652,7 +652,7 @@ async def complete_address_details(conn: SearchConnection, results: List[BaseRes
 
     parent_lookup_ids = list(filter(lambda e: e['pid'] != e['lid'], lookup_ids))
     if parent_lookup_ids:
-        ltab = sa.func.json_array_elements(sa.type_coerce(parent_lookup_ids, sa.JSON))\
+        ltab = sa.func.JsonArrayEach(sa.type_coerce(parent_lookup_ids, sa.JSON))\
                  .table_valued(sa.column('value', type_=sa.JSON)) # type: ignore[no-untyped-call]
         sql = sa.select(ltab.c.value['pid'].as_integer().label('src_place_id'),
                         t.c.place_id, t.c.osm_type, t.c.osm_id, t.c.name,
@@ -687,14 +687,10 @@ def _placex_select_address_row(conn: SearchConnection,
     return sa.select(t.c.place_id, t.c.osm_type, t.c.osm_id, t.c.name,
                      t.c.class_.label('class'), t.c.type,
                      t.c.admin_level, t.c.housenumber,
-                     sa.literal_column("""ST_GeometryType(geometry) in
-                                        ('ST_Polygon','ST_MultiPolygon')""").label('fromarea'),
+                     t.c.geometry.is_area().label('fromarea'),
                      t.c.rank_address,
-                     sa.literal_column(
-                         f"""ST_DistanceSpheroid(geometry,
-                                                 'SRID=4326;{centroid.to_wkt()}'::geometry,
-                              'SPHEROID["WGS 84",6378137,298.257223563, AUTHORITY["EPSG","7030"]]')
-                         """).label('distance'))
+                     t.c.geometry.distance_spheroid(
+                       sa.bindparam('centroid', value=centroid, type_=Geometry)).label('distance'))
 
 
 async def complete_linked_places(conn: SearchConnection, result: BaseResult) -> None:
@@ -728,10 +724,10 @@ async def complete_keywords(conn: SearchConnection, result: BaseResult) -> None:
     sel = sa.select(t.c.word_id, t.c.word_token, t.c.word)
 
     for name_tokens, address_tokens in await conn.execute(sql):
-        for row in await conn.execute(sel.where(t.c.word_id == sa.any_(name_tokens))):
+        for row in await conn.execute(sel.where(t.c.word_id.in_(name_tokens))):
             result.name_keywords.append(WordInfo(*row))
 
-        for row in await conn.execute(sel.where(t.c.word_id == sa.any_(address_tokens))):
+        for row in await conn.execute(sel.where(t.c.word_id.in_(address_tokens))):
             result.address_keywords.append(WordInfo(*row))
 
 

nominatim/api/reverse.py

@@ -19,7 +19,6 @@
 from nominatim.api.logging import log
 from nominatim.api.types import AnyPoint, DataLayer, ReverseDetails, GeometryFormat, Bbox
 from nominatim.db.sqlalchemy_types import Geometry
-import nominatim.db.sqlalchemy_functions as snfn
 
 # In SQLAlchemy expression which compare with NULL need to be expressed with
 # the equal sign.
@@ -85,12 +84,6 @@ def _locate_interpolation(table: SaFromClause) -> SaLabel:
                    else_=0).label('position')
 
 
-def _is_address_point(table: SaFromClause) -> SaColumn:
-    return sa.and_(table.c.rank_address == 30,
-                   sa.or_(table.c.housenumber != None,
-                          table.c.name.has_key('addr:housename')))
-
-
 def _get_closest(*rows: Optional[SaRow]) -> Optional[SaRow]:
     return min(rows, key=lambda row: 1000 if row is None else row.distance)
 
@@ -147,13 +140,13 @@ def _add_geometry_columns(self, sql: SaLambdaSelect, col: SaColumn) -> SaSelect:
             col = sa.func.ST_SimplifyPreserveTopology(col, self.params.geometry_simplification)
 
         if self.params.geometry_output & GeometryFormat.GEOJSON:
-            out.append(sa.func.ST_AsGeoJSON(col).label('geometry_geojson'))
+            out.append(sa.func.ST_AsGeoJSON(col, 7).label('geometry_geojson'))
         if self.params.geometry_output & GeometryFormat.TEXT:
             out.append(sa.func.ST_AsText(col).label('geometry_text'))
         if self.params.geometry_output & GeometryFormat.KML:
-            out.append(sa.func.ST_AsKML(col).label('geometry_kml'))
+            out.append(sa.func.ST_AsKML(col, 7).label('geometry_kml'))
         if self.params.geometry_output & GeometryFormat.SVG:
-            out.append(sa.func.ST_AsSVG(col).label('geometry_svg'))
+            out.append(sa.func.ST_AsSVG(col, 0, 7).label('geometry_svg'))
 
         return sql.add_columns(*out)
 
@@ -204,7 +197,7 @@ async def _find_closest_street_or_poi(self, distance: float) -> Optional[SaRow]:
             max_rank = min(29, self.max_rank)
             restrict.append(lambda: no_index(t.c.rank_address).between(26, max_rank))
             if self.max_rank == 30:
-                restrict.append(lambda: _is_address_point(t))
+                restrict.append(lambda: sa.func.IsAddressPoint(t))
         if self.layer_enabled(DataLayer.POI) and self.max_rank == 30:
             restrict.append(lambda: sa.and_(no_index(t.c.rank_search) == 30,
                                             t.c.class_.not_in(('place', 'building')),
@@ -228,7 +221,7 @@ async def _find_housenumber_for_street(self, parent_place_id: int) -> Optional[S
         sql: SaLambdaSelect = sa.lambda_stmt(lambda: _select_from_placex(t)
                 .where(t.c.geometry.ST_DWithin(WKT_PARAM, 0.001))
                 .where(t.c.parent_place_id == parent_place_id)
-                .where(_is_address_point(t))
+                .where(sa.func.IsAddressPoint(t))
                 .where(t.c.indexed_status == 0)
                 .where(t.c.linked_place_id == None)
                 .order_by('distance')
@@ -371,7 +364,7 @@ def _base_query() -> SaSelect:
             inner = sa.select(t, sa.literal(0.0).label('distance'))\
                       .where(t.c.rank_search.between(5, MAX_RANK_PARAM))\
                       .where(t.c.geometry.intersects(WKT_PARAM))\
-                      .where(snfn.select_index_placex_geometry_reverse_lookuppolygon('placex'))\
+                      .where(sa.func.PlacexGeometryReverseLookuppolygon())\
                       .order_by(sa.desc(t.c.rank_search))\
                       .limit(50)\
                       .subquery('area')
@@ -401,10 +394,7 @@ def _place_inside_area_query() -> SaSelect:
                       .where(t.c.rank_search > address_rank)\
                       .where(t.c.rank_search <= MAX_RANK_PARAM)\
                       .where(t.c.indexed_status == 0)\
-                      .where(snfn.select_index_placex_geometry_reverse_lookupplacenode('placex'))\
-                      .where(t.c.geometry
-                                .ST_Buffer(sa.func.reverse_place_diameter(t.c.rank_search))
-                                .intersects(WKT_PARAM))\
+                      .where(sa.func.IntersectsReverseDistance(t, WKT_PARAM))\
                       .order_by(sa.desc(t.c.rank_search))\
                       .limit(50)\
                       .subquery('places')
@@ -413,7 +403,7 @@ def _place_inside_area_query() -> SaSelect:
                 return _select_from_placex(inner, False)\
                     .join(touter, touter.c.geometry.ST_Contains(inner.c.geometry))\
                     .where(touter.c.place_id == address_id)\
-                    .where(inner.c.distance < sa.func.reverse_place_diameter(inner.c.rank_search))\
+                    .where(sa.func.IsBelowReverseDistance(inner.c.distance, inner.c.rank_search))\
                     .order_by(sa.desc(inner.c.rank_search), inner.c.distance)\
                     .limit(1)
 
@@ -440,10 +430,9 @@ async def _lookup_area_others(self) -> Optional[SaRow]:
                   .where(t.c.indexed_status == 0)\
                   .where(t.c.linked_place_id == None)\
                   .where(self._filter_by_layer(t))\
-                  .where(t.c.geometry
-                                .ST_Buffer(sa.func.reverse_place_diameter(t.c.rank_search))
-                                .intersects(WKT_PARAM))\
+                  .where(t.c.geometry.intersects(sa.func.ST_Expand(WKT_PARAM, 0.007)))\
                   .order_by(sa.desc(t.c.rank_search))\
+                  .order_by('distance')\
                   .limit(50)\
                   .subquery()
 
@@ -514,16 +503,13 @@ def _base_query() -> SaSelect:
                       .where(t.c.rank_search <= MAX_RANK_PARAM)\
                       .where(t.c.indexed_status == 0)\
                       .where(t.c.country_code.in_(ccodes))\
-                      .where(snfn.select_index_placex_geometry_reverse_lookupplacenode('placex'))\
-                      .where(t.c.geometry
-                                .ST_Buffer(sa.func.reverse_place_diameter(t.c.rank_search))
-                                .intersects(WKT_PARAM))\
+                      .where(sa.func.IntersectsReverseDistance(t, WKT_PARAM))\
                       .order_by(sa.desc(t.c.rank_search))\
                       .limit(50)\
                       .subquery('area')
 
                 return _select_from_placex(inner, False)\
-                    .where(inner.c.distance < sa.func.reverse_place_diameter(inner.c.rank_search))\
+                    .where(sa.func.IsBelowReverseDistance(inner.c.distance, inner.c.rank_search))\
                     .order_by(sa.desc(inner.c.rank_search), inner.c.distance)\
                     .limit(1)