From 65ce6ebd58b6afbd2256c0a98a290f071dc021f1 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 27 Jun 2024 21:26:12 +0200 Subject: [PATCH] reduce from 3 to 2 packages --- CMakeLists.txt | 20 +++---- Makefile | 8 +-- docs/admin/Installation.md | 2 +- docs/library/Configuration.md | 2 +- packaging/nominatim-api/extra_src/paths.py | 15 +++++ packaging/nominatim-api/pyproject.toml | 18 +++++- .../settings | 0 packaging/nominatim-core/COPYING | 1 - packaging/nominatim-core/README.md | 24 -------- packaging/nominatim-core/data | 1 - packaging/nominatim-core/pyproject.toml | 60 ------------------- packaging/nominatim-core/src | 1 - packaging/nominatim-db/data | 1 + .../extra_src/nominatim_db}/paths.py | 0 .../{nominatim-core => nominatim-db}/lib-sql | 0 packaging/nominatim-db/pyproject.toml | 30 +++++++++- packaging/nominatim-db/settings | 1 + src/nominatim_api/__init__.py | 4 +- src/nominatim_api/config.py | 12 ++++ src/nominatim_api/connection.py | 6 +- src/nominatim_api/core.py | 8 +-- .../errors.py | 0 src/nominatim_api/lookup.py | 2 +- src/nominatim_api/results.py | 4 +- src/nominatim_api/reverse.py | 6 +- src/nominatim_api/search/db_search_fields.py | 4 +- src/nominatim_api/search/db_search_lookups.py | 4 +- src/nominatim_api/search/db_searches.py | 6 +- src/nominatim_api/search/icu_tokenizer.py | 4 +- src/nominatim_api/search/legacy_tokenizer.py | 2 +- src/nominatim_api/server/falcon/server.py | 2 +- src/nominatim_api/server/starlette/server.py | 2 +- .../sql}/async_core_library.py | 0 src/nominatim_api/sql/sqlalchemy_functions.py | 2 +- .../sql}/sqlalchemy_schema.py | 0 .../sql}/sqlalchemy_types/__init__.py | 0 .../sql}/sqlalchemy_types/geometry.py | 0 .../sql}/sqlalchemy_types/int_array.py | 0 .../sql}/sqlalchemy_types/json.py | 0 .../sql}/sqlalchemy_types/key_value.py | 0 src/nominatim_api/status.py | 6 +- src/nominatim_api/types.py | 2 +- src/nominatim_api/typing.py | 39 ++++++++++++ .../utils}/__init__.py | 0 .../utils/json_writer.py | 0 src/nominatim_api/v1/format.py | 2 +- src/nominatim_api/v1/format_json.py | 2 +- src/nominatim_api/v1/server_glue.py | 4 +- src/nominatim_api/version.py | 6 -- src/nominatim_core/py.typed | 0 src/nominatim_core/version.py | 57 ------------------ src/nominatim_db/cli.py | 4 +- src/nominatim_db/clicmd/admin.py | 4 +- src/nominatim_db/clicmd/api.py | 2 +- src/nominatim_db/clicmd/args.py | 6 +- src/nominatim_db/clicmd/convert.py | 2 +- src/nominatim_db/clicmd/export.py | 7 ++- src/nominatim_db/clicmd/freeze.py | 2 +- src/nominatim_db/clicmd/index.py | 4 +- src/nominatim_db/clicmd/refresh.py | 4 +- src/nominatim_db/clicmd/replication.py | 6 +- src/nominatim_db/clicmd/setup.py | 8 +-- src/nominatim_db/clicmd/special_phrases.py | 4 +- .../config.py | 6 +- src/nominatim_db/data/country_info.py | 8 +-- src/nominatim_db/data/postcode_format.py | 2 +- .../db/__init__.py | 0 .../db/async_connection.py | 0 .../db/connection.py | 0 .../db/properties.py | 0 .../db/sql_preprocessor.py | 0 .../db/status.py | 0 .../db/utils.py | 0 src/nominatim_db/errors.py | 14 +++++ src/nominatim_db/indexer/indexer.py | 6 +- src/nominatim_db/indexer/runners.py | 4 +- src/{nominatim_core => nominatim_db}/paths.py | 0 src/nominatim_db/tokenizer/base.py | 6 +- src/nominatim_db/tokenizer/factory.py | 8 +-- src/nominatim_db/tokenizer/icu_rule_loader.py | 8 +-- src/nominatim_db/tokenizer/icu_tokenizer.py | 8 +-- .../tokenizer/legacy_tokenizer.py | 12 ++-- src/nominatim_db/tokenizer/place_sanitizer.py | 4 +- src/nominatim_db/tokenizer/sanitizers/base.py | 2 +- .../tokenizer/sanitizers/config.py | 2 +- .../tokenizer/token_analysis/base.py | 2 +- .../token_analysis/config_variants.py | 4 +- .../tokenizer/token_analysis/generic.py | 2 +- .../token_analysis/generic_mutation.py | 2 +- src/nominatim_db/tools/add_osm_data.py | 4 +- src/nominatim_db/tools/admin.py | 8 +-- src/nominatim_db/tools/check_database.py | 8 +-- src/nominatim_db/tools/collect_os_info.py | 4 +- src/nominatim_db/tools/convert_sqlite.py | 4 +- src/nominatim_db/tools/database_import.py | 10 ++-- src/nominatim_db/tools/exec_utils.py | 4 +- src/nominatim_db/tools/freeze.py | 2 +- src/nominatim_db/tools/migration.py | 8 +-- src/nominatim_db/tools/postcodes.py | 4 +- src/nominatim_db/tools/refresh.py | 8 +-- src/nominatim_db/tools/replication.py | 6 +- .../tools/special_phrases/sp_csv_loader.py | 2 +- .../tools/special_phrases/sp_importer.py | 6 +- .../tools/special_phrases/sp_wiki_loader.py | 4 +- src/nominatim_db/tools/tiger_data.py | 10 ++-- .../typing.py | 23 ------- .../utils/__init__.py | 0 .../utils/centroid.py | 0 .../utils/url_utils.py | 4 +- src/nominatim_db/version.py | 48 ++++++++++++++- test/bdd/steps/nominatim_environment.py | 4 +- test/python/api/conftest.py | 2 +- test/python/api/fake_adaptor.py | 2 +- test/python/api/test_api_status.py | 2 +- test/python/api/test_api_types.py | 2 +- test/python/cli/test_cmd_import.py | 8 +-- test/python/cli/test_cmd_replication.py | 2 +- test/python/config/test_config.py | 4 +- test/python/config/test_config_load_module.py | 2 +- test/python/conftest.py | 6 +- test/python/db/test_async_connection.py | 2 +- test/python/db/test_connection.py | 2 +- test/python/db/test_properties.py | 2 +- test/python/db/test_sql_preprocessor.py | 2 +- test/python/db/test_status.py | 46 +++++++------- test/python/db/test_utils.py | 4 +- test/python/dummy_tokenizer.py | 2 +- test/python/mocks.py | 2 +- .../sanitizers/test_sanitizer_config.py | 2 +- .../sanitizers/test_split_name_list.py | 2 +- test/python/tokenizer/test_factory.py | 4 +- test/python/tokenizer/test_icu.py | 4 +- test/python/tokenizer/test_icu_rule_loader.py | 2 +- test/python/tokenizer/test_legacy.py | 4 +- test/python/tokenizer/test_place_sanitizer.py | 2 +- .../token_analysis/test_analysis_postcodes.py | 2 +- .../tokenizer/token_analysis/test_generic.py | 2 +- .../token_analysis/test_generic_mutation.py | 2 +- test/python/tools/test_admin.py | 4 +- test/python/tools/test_database_import.py | 2 +- test/python/tools/test_exec_utils.py | 2 +- .../tools/test_import_special_phrases.py | 2 +- test/python/tools/test_migration.py | 2 +- test/python/tools/test_replication.py | 4 +- test/python/tools/test_sp_csv_loader.py | 2 +- test/python/tools/test_tiger_data.py | 2 +- test/python/utils/test_centroid.py | 2 +- test/python/utils/test_json_writer.py | 2 +- vagrant/Install-on-Ubuntu-24.sh | 2 +- 149 files changed, 417 insertions(+), 421 deletions(-) create mode 100644 packaging/nominatim-api/extra_src/paths.py rename packaging/{nominatim-core => nominatim-api}/settings (100%) delete mode 120000 packaging/nominatim-core/COPYING delete mode 100644 packaging/nominatim-core/README.md delete mode 120000 packaging/nominatim-core/data delete mode 100644 packaging/nominatim-core/pyproject.toml delete mode 120000 packaging/nominatim-core/src create mode 120000 packaging/nominatim-db/data rename packaging/{nominatim-core/extra_src/nominatim_core => nominatim-db/extra_src/nominatim_db}/paths.py (100%) rename packaging/{nominatim-core => nominatim-db}/lib-sql (100%) create mode 120000 packaging/nominatim-db/settings create mode 100644 src/nominatim_api/config.py rename src/{nominatim_core => nominatim_api}/errors.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/async_core_library.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_schema.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/__init__.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/geometry.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/int_array.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/json.py (100%) rename src/{nominatim_core/db => nominatim_api/sql}/sqlalchemy_types/key_value.py (100%) create mode 100644 src/nominatim_api/typing.py rename src/{nominatim_core => nominatim_api/utils}/__init__.py (100%) rename src/{nominatim_core => nominatim_api}/utils/json_writer.py (100%) delete mode 100644 src/nominatim_core/py.typed delete mode 100644 src/nominatim_core/version.py rename src/{nominatim_core => nominatim_db}/config.py (98%) rename src/{nominatim_core => nominatim_db}/db/__init__.py (100%) rename src/{nominatim_core => nominatim_db}/db/async_connection.py (100%) rename src/{nominatim_core => nominatim_db}/db/connection.py (100%) rename src/{nominatim_core => nominatim_db}/db/properties.py (100%) rename src/{nominatim_core => nominatim_db}/db/sql_preprocessor.py (100%) rename src/{nominatim_core => nominatim_db}/db/status.py (100%) rename src/{nominatim_core => nominatim_db}/db/utils.py (100%) create mode 100644 src/nominatim_db/errors.py rename src/{nominatim_core => nominatim_db}/paths.py (100%) rename src/{nominatim_core => nominatim_db}/typing.py (65%) rename src/{nominatim_core => nominatim_db}/utils/__init__.py (100%) rename src/{nominatim_core => nominatim_db}/utils/centroid.py (100%) rename src/{nominatim_core => nominatim_db}/utils/url_utils.py (87%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fa4d1b47e..01a71a3c59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,23 +221,23 @@ if (BUILD_IMPORTER) DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME nominatim) - foreach (submodule nominatim_core nominatim_db nominatim_api) + if (EXISTS ${PHP_BIN}) + configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py.tmpl paths-py.installed) + else() + configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py-no-php.tmpl paths-py.installed) + endif() + + foreach (submodule nominatim_db nominatim_api) install(DIRECTORY src/${submodule} DESTINATION ${NOMINATIM_LIBDIR}/lib-python FILES_MATCHING PATTERN "*.py" PATTERN "paths.py" EXCLUDE PATTERN __pycache__ EXCLUDE) + install(FILES ${PROJECT_BINARY_DIR}/paths-py.installed + DESTINATION ${NOMINATIM_LIBDIR}/lib-python/${submodule} + RENAME paths.py) endforeach() - if (EXISTS ${PHP_BIN}) - configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py.tmpl paths-py.installed) - else() - configure_file(${PROJECT_SOURCE_DIR}/cmake/paths-py-no-php.tmpl paths-py.installed) - endif() - install(FILES ${PROJECT_BINARY_DIR}/paths-py.installed - DESTINATION ${NOMINATIM_LIBDIR}/lib-python/nominatim_core - RENAME paths.py) - install(DIRECTORY lib-sql DESTINATION ${NOMINATIM_LIBDIR}) install(FILES ${COUNTRY_GRID_FILE} diff --git a/Makefile b/Makefile index 0e09a884ff..dae45322a0 100644 --- a/Makefile +++ b/Makefile @@ -2,10 +2,10 @@ all: # Building of wheels -build: build-core build-db build-api +build: clean-build build-db build-api -build-core: - python3 -m build packaging/nominatim-core --outdir dist/ +clean-build: + rm -f dist/* build-db: python3 -m build packaging/nominatim-db --outdir dist/ @@ -29,4 +29,4 @@ lint: bdd: cd test/bdd; behave -DREMOVE_TEMPLATE=1 -.PHONY: tests mypy pytest lint bdd build build-core build-db build-api +.PHONY: tests mypy pytest lint bdd build clean-build build-db build-api diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index cb0df5c140..a2f1a084c5 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -162,7 +162,7 @@ Nominatim is easiest to run from its own virtual environment. To create one, run To install Nominatim directly from the source tree into the virtual environment, run: - /srv/nominatim-venv/bin/pip install packaging/nominatim-{core,db,api} + /srv/nominatim-venv/bin/pip install packaging/nominatim-{db,api} #### Building in legacy CMake mode diff --git a/docs/library/Configuration.md b/docs/library/Configuration.md index bf093166b7..e13470e9fc 100644 --- a/docs/library/Configuration.md +++ b/docs/library/Configuration.md @@ -20,7 +20,7 @@ configuration parameters, see the [Configuration page](../customize/Settings.md) ## `Configuration` class -::: nominatim_core.config.Configuration +::: nominatim_api.Configuration options: members: - get_bool diff --git a/packaging/nominatim-api/extra_src/paths.py b/packaging/nominatim-api/extra_src/paths.py new file mode 100644 index 0000000000..6131319ca8 --- /dev/null +++ b/packaging/nominatim-api/extra_src/paths.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Path settings for extra data used by Nominatim. +""" +from pathlib import Path + +PHPLIB_DIR = None +DATA_DIR = None +SQLLIB_DIR = None +CONFIG_DIR = (Path(__file__) / '..' / 'resources' / 'settings').resolve() diff --git a/packaging/nominatim-api/pyproject.toml b/packaging/nominatim-api/pyproject.toml index 7d270021c2..4b268b1653 100644 --- a/packaging/nominatim-api/pyproject.toml +++ b/packaging/nominatim-api/pyproject.toml @@ -14,7 +14,8 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "nominatim-core", + "python-dotenv", + "pyYAML>=5.1", "SQLAlchemy>=1.4.31", "psycopg", "PyICU" @@ -34,8 +35,21 @@ path = "src/nominatim_api/version.py" pattern = "NOMINATIM_API_VERSION = '(?P[^']+)'" [tool.hatch.build.targets.sdist] -include = ["src/nominatim_api"] +include = [ + "src/nominatim_api", + "src/nominatim_db/config.py", + "settings", + "extra_src/paths.py" +] + +exclude = [ + "src/nominatim_api/config.py" +] [tool.hatch.build.targets.wheel] packages = ["src/nominatim_api"] +[tool.hatch.build.targets.wheel.force-include] +"src/nominatim_db/config.py" = "nominatim_api/config.py" +"extra_src/paths.py" = "nominatim_api/paths.py" +"settings" = "nominatim_api/resources/settings" diff --git a/packaging/nominatim-core/settings b/packaging/nominatim-api/settings similarity index 100% rename from packaging/nominatim-core/settings rename to packaging/nominatim-api/settings diff --git a/packaging/nominatim-core/COPYING b/packaging/nominatim-core/COPYING deleted file mode 120000 index 7d29222e4c..0000000000 --- a/packaging/nominatim-core/COPYING +++ /dev/null @@ -1 +0,0 @@ -../../COPYING \ No newline at end of file diff --git a/packaging/nominatim-core/README.md b/packaging/nominatim-core/README.md deleted file mode 100644 index 127303fcf9..0000000000 --- a/packaging/nominatim-core/README.md +++ /dev/null @@ -1,24 +0,0 @@ -Nominatim - Core Package -========= - -Nominatim is a tool to search OpenStreetMap data -by name and address (geocoding) and to generate synthetic addresses of -OSM points (reverse geocoding). - -This is the core pacakage containing resources and code shared by -Nominatim's frontend `nominatim-api` and backend `nominatim-db`. You -usually don't want to install this package directly. - -Documentation -============= - -The documentation of the latest development version is in the -`docs/` subdirectory. A HTML version can be found at -https://nominatim.org/release-docs/develop/ . - -License -======= - -The Python source code is available under a GPL license version 3 or later. -The Lua configuration files for osm2pgsql are released under the -Apache License, Version 2.0. All other files are under a GPLv2 license. diff --git a/packaging/nominatim-core/data b/packaging/nominatim-core/data deleted file mode 120000 index e67b455909..0000000000 --- a/packaging/nominatim-core/data +++ /dev/null @@ -1 +0,0 @@ -../../data \ No newline at end of file diff --git a/packaging/nominatim-core/pyproject.toml b/packaging/nominatim-core/pyproject.toml deleted file mode 100644 index 9c7af6514a..0000000000 --- a/packaging/nominatim-core/pyproject.toml +++ /dev/null @@ -1,60 +0,0 @@ -[project] -name = "nominatim-core" -description = "A tool for building a database of OpenStreetMap for geocoding and for searching the database. Base package for common resources for the project." -readme = "README.md" -requires-python = ">=3.7" -license = 'GPL-3.0-or-later' -maintainers = [ - { name = "Sarah Hoffmann", email = "lonvia@denofr.de" } -] -keywords = [ "geocoding", "OpenStreetMap", "search" ] -classifiers = [ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License (GPL)", - "Operating System :: OS Independent", -] -dependencies = [ - "python-dotenv", - "jinja2", - "pyYAML>=5.1", - "datrie" -] -dynamic = ["version"] - -[project.urls] -Homepage = "https://nominatim.org" -Issues = "https://github.com/osm-search/Nominatim/issues" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.version] -source = "code" -path = "src/nominatim_core/version.py" -expression = "NOMINATIM_CORE_VERSION" - -[tool.hatch.build.targets.sdist] -include = [ - "src/nominatim_core", - "lib-sql/**.sql", - "settings", - "data/words.sql", - "extra_src/nominatim_core/paths.py" -] -artifacts = [ - "data/country_osm_grid.sql.gz" -] -exclude = [ - "src/nominatim_core/paths.py" -] - -[tool.hatch.build.targets.wheel] -packages = ["src/nominatim_core"] - -[tool.hatch.build.targets.wheel.force-include] -"lib-sql" = "nominatim_core/resources/lib-sql" -"settings" = "nominatim_core/resources/settings" -"data/country_osm_grid.sql.gz" = "nominatim_core/resources/country_osm_grid.sql.gz" -"data/words.sql" = "nominatim_core/resources/words.sql" -"extra_src/nominatim_core/paths.py" = "nominatim_core/paths.py" diff --git a/packaging/nominatim-core/src b/packaging/nominatim-core/src deleted file mode 120000 index 929cb3dc9b..0000000000 --- a/packaging/nominatim-core/src +++ /dev/null @@ -1 +0,0 @@ -../../src \ No newline at end of file diff --git a/packaging/nominatim-db/data b/packaging/nominatim-db/data new file mode 120000 index 0000000000..b8fb52e773 --- /dev/null +++ b/packaging/nominatim-db/data @@ -0,0 +1 @@ +../../data/ \ No newline at end of file diff --git a/packaging/nominatim-core/extra_src/nominatim_core/paths.py b/packaging/nominatim-db/extra_src/nominatim_db/paths.py similarity index 100% rename from packaging/nominatim-core/extra_src/nominatim_core/paths.py rename to packaging/nominatim-db/extra_src/nominatim_db/paths.py diff --git a/packaging/nominatim-core/lib-sql b/packaging/nominatim-db/lib-sql similarity index 100% rename from packaging/nominatim-core/lib-sql rename to packaging/nominatim-db/lib-sql diff --git a/packaging/nominatim-db/pyproject.toml b/packaging/nominatim-db/pyproject.toml index 312f1c8126..69b863c2a9 100644 --- a/packaging/nominatim-db/pyproject.toml +++ b/packaging/nominatim-db/pyproject.toml @@ -14,8 +14,11 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "nominatim-core", "psycopg2-binary", + "python-dotenv", + "jinja2", + "pyYAML>=5.1", + "datrie", "psutil", "PyICU" ] @@ -24,6 +27,7 @@ dynamic = ["version"] [project.urls] Homepage = "https://nominatim.org" Issues = "https://github.com/osm-search/Nominatim/issues" +Documentation = "https://nominatim.org/release-docs/latest/" [build-system] requires = ["hatchling"] @@ -34,10 +38,32 @@ path = "src/nominatim_db/version.py" pattern = "NOMINATIM_VERSION = parse_version.'(?P[^-]+)" [tool.hatch.build.targets.sdist] -include = ["src/nominatim_db", "scripts"] +include = [ + "src/nominatim_db", + "scripts", + "lib-sql/**/*.sql", + "settings", + "data/words.sql", + "extra_src/nominatim_db/paths.py" +] + +artifacts = [ + "data/country_osm_grid.sql.gz" +] + +exclude = [ + "src/nominatim_db/paths.py" +] [tool.hatch.build.targets.wheel] packages = ["src/nominatim_db"] [tool.hatch.build.targets.wheel.shared-scripts] "scripts" = "/" + +[tool.hatch.build.targets.wheel.force-include] +"lib-sql" = "nominatim_db/resources/lib-sql" +"settings" = "nominatim_db/resources/settings" +"data/country_osm_grid.sql.gz" = "nominatim_db/resources/country_osm_grid.sql.gz" +"data/words.sql" = "nominatim_db/resources/words.sql" +"extra_src/nominatim_db/paths.py" = "nominatim_db/paths.py" diff --git a/packaging/nominatim-db/settings b/packaging/nominatim-db/settings new file mode 120000 index 0000000000..51326e662f --- /dev/null +++ b/packaging/nominatim-db/settings @@ -0,0 +1 @@ +../../settings/ \ No newline at end of file diff --git a/src/nominatim_api/__init__.py b/src/nominatim_api/__init__.py index 0f02f979d6..4f4714479a 100644 --- a/src/nominatim_api/__init__.py +++ b/src/nominatim_api/__init__.py @@ -14,8 +14,8 @@ # See also https://github.com/PyCQA/pylint/issues/6006 # pylint: disable=useless-import-alias -from nominatim_core.errors import (UsageError as UsageError) -from nominatim_core.config import (Configuration as Configuration) +from .errors import (UsageError as UsageError) +from .config import (Configuration as Configuration) from .core import (NominatimAPI as NominatimAPI, NominatimAPIAsync as NominatimAPIAsync) diff --git a/src/nominatim_api/config.py b/src/nominatim_api/config.py new file mode 100644 index 0000000000..18afda6687 --- /dev/null +++ b/src/nominatim_api/config.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. + +# This file is just a placeholder to make the config module available +# during development. It will be replaced by nominatim_db/config.py on +# installation. +# pylint: skip-file +from nominatim_db.config import * diff --git a/src/nominatim_api/connection.py b/src/nominatim_api/connection.py index 66b00ac813..167ffaa454 100644 --- a/src/nominatim_api/connection.py +++ b/src/nominatim_api/connection.py @@ -14,9 +14,9 @@ import sqlalchemy as sa from sqlalchemy.ext.asyncio import AsyncConnection -from nominatim_core.typing import SaFromClause -from nominatim_core.db.sqlalchemy_schema import SearchTables -from nominatim_core.db.sqlalchemy_types import Geometry +from .typing import SaFromClause +from .sql.sqlalchemy_schema import SearchTables +from .sql.sqlalchemy_types import Geometry from .logging import log T = TypeVar('T') diff --git a/src/nominatim_api/core.py b/src/nominatim_api/core.py index 6f91e7a8e4..632c97a7a6 100644 --- a/src/nominatim_api/core.py +++ b/src/nominatim_api/core.py @@ -16,10 +16,10 @@ import sqlalchemy as sa import sqlalchemy.ext.asyncio as sa_asyncio -from nominatim_core.errors import UsageError -from nominatim_core.db.sqlalchemy_schema import SearchTables -from nominatim_core.db.async_core_library import PGCORE_LIB, PGCORE_ERROR -from nominatim_core.config import Configuration +from .errors import UsageError +from .sql.sqlalchemy_schema import SearchTables +from .sql.async_core_library import PGCORE_LIB, PGCORE_ERROR +from .config import Configuration from .sql import sqlite_functions, sqlalchemy_functions #pylint: disable=unused-import from .connection import SearchConnection from .status import get_status, StatusResult diff --git a/src/nominatim_core/errors.py b/src/nominatim_api/errors.py similarity index 100% rename from src/nominatim_core/errors.py rename to src/nominatim_api/errors.py diff --git a/src/nominatim_api/lookup.py b/src/nominatim_api/lookup.py index e451edbee0..34739171d8 100644 --- a/src/nominatim_api/lookup.py +++ b/src/nominatim_api/lookup.py @@ -12,7 +12,7 @@ import sqlalchemy as sa -from nominatim_core.typing import SaColumn, SaRow, SaSelect +from .typing import SaColumn, SaRow, SaSelect from .connection import SearchConnection from .logging import log from . import types as ntyp diff --git a/src/nominatim_api/results.py b/src/nominatim_api/results.py index 4256e0b7f8..805527a7b5 100644 --- a/src/nominatim_api/results.py +++ b/src/nominatim_api/results.py @@ -18,8 +18,8 @@ import sqlalchemy as sa -from nominatim_core.typing import SaSelect, SaRow -from nominatim_core.db.sqlalchemy_types import Geometry +from .typing import SaSelect, SaRow +from .sql.sqlalchemy_types import Geometry from .types import Point, Bbox, LookupDetails from .connection import SearchConnection from .logging import log diff --git a/src/nominatim_api/reverse.py b/src/nominatim_api/reverse.py index 83b0983af6..20270656f9 100644 --- a/src/nominatim_api/reverse.py +++ b/src/nominatim_api/reverse.py @@ -12,9 +12,9 @@ import sqlalchemy as sa -from nominatim_core.typing import SaColumn, SaSelect, SaFromClause, SaLabel, SaRow,\ - SaBind, SaLambdaSelect -from nominatim_core.db.sqlalchemy_types import Geometry +from .typing import SaColumn, SaSelect, SaFromClause, SaLabel, SaRow,\ + SaBind, SaLambdaSelect +from .sql.sqlalchemy_types import Geometry from .connection import SearchConnection from . import results as nres from .logging import log diff --git a/src/nominatim_api/search/db_search_fields.py b/src/nominatim_api/search/db_search_fields.py index 8f674ecc9f..95b2b4a7d8 100644 --- a/src/nominatim_api/search/db_search_fields.py +++ b/src/nominatim_api/search/db_search_fields.py @@ -12,8 +12,8 @@ import sqlalchemy as sa -from nominatim_core.typing import SaFromClause, SaColumn, SaExpression -from nominatim_core.utils.json_writer import JsonWriter +from ..typing import SaFromClause, SaColumn, SaExpression +from ..utils.json_writer import JsonWriter from .query import Token from . import db_search_lookups as lookups diff --git a/src/nominatim_api/search/db_search_lookups.py b/src/nominatim_api/search/db_search_lookups.py index faabd82794..712cd89447 100644 --- a/src/nominatim_api/search/db_search_lookups.py +++ b/src/nominatim_api/search/db_search_lookups.py @@ -12,8 +12,8 @@ import sqlalchemy as sa from sqlalchemy.ext.compiler import compiles -from nominatim_core.typing import SaFromClause -from nominatim_core.db.sqlalchemy_types import IntArray +from ..typing import SaFromClause +from ..sql.sqlalchemy_types import IntArray # pylint: disable=consider-using-f-string diff --git a/src/nominatim_api/search/db_searches.py b/src/nominatim_api/search/db_searches.py index f5c431460d..35c063fc93 100644 --- a/src/nominatim_api/search/db_searches.py +++ b/src/nominatim_api/search/db_searches.py @@ -12,9 +12,9 @@ import sqlalchemy as sa -from nominatim_core.typing import SaFromClause, SaScalarSelect, SaColumn, \ - SaExpression, SaSelect, SaLambdaSelect, SaRow, SaBind -from nominatim_core.db.sqlalchemy_types import Geometry, IntArray +from ..typing import SaFromClause, SaScalarSelect, SaColumn, \ + SaExpression, SaSelect, SaLambdaSelect, SaRow, SaBind +from ..sql.sqlalchemy_types import Geometry, IntArray from ..connection import SearchConnection from ..types import SearchDetails, DataLayer, GeometryFormat, Bbox from .. import results as nres diff --git a/src/nominatim_api/search/icu_tokenizer.py b/src/nominatim_api/search/icu_tokenizer.py index 426656797c..971e95beec 100644 --- a/src/nominatim_api/search/icu_tokenizer.py +++ b/src/nominatim_api/search/icu_tokenizer.py @@ -16,8 +16,8 @@ import sqlalchemy as sa -from nominatim_core.typing import SaRow -from nominatim_core.db.sqlalchemy_types import Json +from ..typing import SaRow +from ..sql.sqlalchemy_types import Json from ..connection import SearchConnection from ..logging import log from ..search import query as qmod diff --git a/src/nominatim_api/search/legacy_tokenizer.py b/src/nominatim_api/search/legacy_tokenizer.py index b49d3ae6b2..ecb0bbfe50 100644 --- a/src/nominatim_api/search/legacy_tokenizer.py +++ b/src/nominatim_api/search/legacy_tokenizer.py @@ -14,7 +14,7 @@ import sqlalchemy as sa -from nominatim_core.typing import SaRow +from ..typing import SaRow from ..connection import SearchConnection from ..logging import log from . import query as qmod diff --git a/src/nominatim_api/server/falcon/server.py b/src/nominatim_api/server/falcon/server.py index 851b096437..bc9850b242 100644 --- a/src/nominatim_api/server/falcon/server.py +++ b/src/nominatim_api/server/falcon/server.py @@ -14,7 +14,7 @@ from falcon.asgi import App, Request, Response -from nominatim_core.config import Configuration +from ...config import Configuration from ...core import NominatimAPIAsync from ... import v1 as api_impl from ... import logging as loglib diff --git a/src/nominatim_api/server/starlette/server.py b/src/nominatim_api/server/starlette/server.py index dd35cd6e9e..5f5cf05507 100644 --- a/src/nominatim_api/server/starlette/server.py +++ b/src/nominatim_api/server/starlette/server.py @@ -21,7 +21,7 @@ from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from starlette.middleware.cors import CORSMiddleware -from nominatim_core.config import Configuration +from ...config import Configuration from ...core import NominatimAPIAsync from ... import v1 as api_impl from ... import logging as loglib diff --git a/src/nominatim_core/db/async_core_library.py b/src/nominatim_api/sql/async_core_library.py similarity index 100% rename from src/nominatim_core/db/async_core_library.py rename to src/nominatim_api/sql/async_core_library.py diff --git a/src/nominatim_api/sql/sqlalchemy_functions.py b/src/nominatim_api/sql/sqlalchemy_functions.py index 402027493f..7abe7d3f33 100644 --- a/src/nominatim_api/sql/sqlalchemy_functions.py +++ b/src/nominatim_api/sql/sqlalchemy_functions.py @@ -13,7 +13,7 @@ import sqlalchemy as sa from sqlalchemy.ext.compiler import compiles -from nominatim_core.typing import SaColumn +from ..typing import SaColumn # pylint: disable=all diff --git a/src/nominatim_core/db/sqlalchemy_schema.py b/src/nominatim_api/sql/sqlalchemy_schema.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_schema.py rename to src/nominatim_api/sql/sqlalchemy_schema.py diff --git a/src/nominatim_core/db/sqlalchemy_types/__init__.py b/src/nominatim_api/sql/sqlalchemy_types/__init__.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/__init__.py rename to src/nominatim_api/sql/sqlalchemy_types/__init__.py diff --git a/src/nominatim_core/db/sqlalchemy_types/geometry.py b/src/nominatim_api/sql/sqlalchemy_types/geometry.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/geometry.py rename to src/nominatim_api/sql/sqlalchemy_types/geometry.py diff --git a/src/nominatim_core/db/sqlalchemy_types/int_array.py b/src/nominatim_api/sql/sqlalchemy_types/int_array.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/int_array.py rename to src/nominatim_api/sql/sqlalchemy_types/int_array.py diff --git a/src/nominatim_core/db/sqlalchemy_types/json.py b/src/nominatim_api/sql/sqlalchemy_types/json.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/json.py rename to src/nominatim_api/sql/sqlalchemy_types/json.py diff --git a/src/nominatim_core/db/sqlalchemy_types/key_value.py b/src/nominatim_api/sql/sqlalchemy_types/key_value.py similarity index 100% rename from src/nominatim_core/db/sqlalchemy_types/key_value.py rename to src/nominatim_api/sql/sqlalchemy_types/key_value.py diff --git a/src/nominatim_api/status.py b/src/nominatim_api/status.py index 614789ed4e..ea88a467d0 100644 --- a/src/nominatim_api/status.py +++ b/src/nominatim_api/status.py @@ -14,7 +14,7 @@ import sqlalchemy as sa from .connection import SearchConnection -from .version import NOMINATIM_API_VERSION, NominatimVersion, parse_version +from .version import NOMINATIM_API_VERSION @dataclasses.dataclass class StatusResult: @@ -24,7 +24,7 @@ class StatusResult: message: str software_version = NOMINATIM_API_VERSION data_updated: Optional[dt.datetime] = None - database_version: Optional[NominatimVersion] = None + database_version: Optional[str] = None async def get_status(conn: SearchConnection) -> StatusResult: @@ -44,7 +44,7 @@ async def get_status(conn: SearchConnection) -> StatusResult: # Database version try: - status.database_version = parse_version(await conn.get_property('database_version')) + status.database_version = await conn.get_property('database_version') except ValueError: pass diff --git a/src/nominatim_api/types.py b/src/nominatim_api/types.py index f2ad4f356e..6c8adeb826 100644 --- a/src/nominatim_api/types.py +++ b/src/nominatim_api/types.py @@ -16,7 +16,7 @@ from struct import unpack from binascii import unhexlify -from nominatim_core.errors import UsageError +from .errors import UsageError from .localization import Locales # pylint: disable=no-member,too-many-boolean-expressions,too-many-instance-attributes diff --git a/src/nominatim_api/typing.py b/src/nominatim_api/typing.py new file mode 100644 index 0000000000..d5570213b4 --- /dev/null +++ b/src/nominatim_api/typing.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Type definitions for typing annotations. + +Complex type definitions are moved here, to keep the source files readable. +""" +from typing import Union, TYPE_CHECKING + +# pylint: disable=missing-class-docstring,useless-import-alias + +StrPath = Union[str, 'os.PathLike[str]'] + +# SQLAlchemy introduced generic types in version 2.0 making typing +# incompatible with older versions. Add wrappers here so we don't have +# to litter the code with bare-string types. + +if TYPE_CHECKING: + from typing import Any + import sqlalchemy as sa + from typing_extensions import (TypeAlias as TypeAlias) +else: + TypeAlias = str + +SaLambdaSelect: TypeAlias = 'Union[sa.Select[Any], sa.StatementLambdaElement]' +SaSelect: TypeAlias = 'sa.Select[Any]' +SaScalarSelect: TypeAlias = 'sa.ScalarSelect[Any]' +SaRow: TypeAlias = 'sa.Row[Any]' +SaColumn: TypeAlias = 'sa.ColumnElement[Any]' +SaExpression: TypeAlias = 'sa.ColumnElement[bool]' +SaLabel: TypeAlias = 'sa.Label[Any]' +SaFromClause: TypeAlias = 'sa.FromClause' +SaSelectable: TypeAlias = 'sa.Selectable' +SaBind: TypeAlias = 'sa.BindParameter[Any]' +SaDialect: TypeAlias = 'sa.Dialect' diff --git a/src/nominatim_core/__init__.py b/src/nominatim_api/utils/__init__.py similarity index 100% rename from src/nominatim_core/__init__.py rename to src/nominatim_api/utils/__init__.py diff --git a/src/nominatim_core/utils/json_writer.py b/src/nominatim_api/utils/json_writer.py similarity index 100% rename from src/nominatim_core/utils/json_writer.py rename to src/nominatim_api/utils/json_writer.py diff --git a/src/nominatim_api/v1/format.py b/src/nominatim_api/v1/format.py index d8faa482c3..e74b61e1cc 100644 --- a/src/nominatim_api/v1/format.py +++ b/src/nominatim_api/v1/format.py @@ -11,7 +11,7 @@ import collections import datetime as dt -from nominatim_core.utils.json_writer import JsonWriter +from ..utils.json_writer import JsonWriter from ..status import StatusResult from ..results import DetailedResult, ReverseResults, SearchResults, \ AddressLines, AddressLine diff --git a/src/nominatim_api/v1/format_json.py b/src/nominatim_api/v1/format_json.py index 2c50dec425..34bb777a2f 100644 --- a/src/nominatim_api/v1/format_json.py +++ b/src/nominatim_api/v1/format_json.py @@ -9,7 +9,7 @@ """ from typing import Mapping, Any, Optional, Tuple, Union -from nominatim_core.utils.json_writer import JsonWriter +from ..utils.json_writer import JsonWriter from ..results import AddressLines, ReverseResults, SearchResults from . import classtypes as cl diff --git a/src/nominatim_api/v1/server_glue.py b/src/nominatim_api/v1/server_glue.py index 67def74e13..c00b580bde 100644 --- a/src/nominatim_api/v1/server_glue.py +++ b/src/nominatim_api/v1/server_glue.py @@ -17,8 +17,8 @@ import sqlalchemy as sa -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration +from ..errors import UsageError +from ..config import Configuration from .. import logging as loglib from ..core import NominatimAPIAsync from .format import dispatch as formatting diff --git a/src/nominatim_api/version.py b/src/nominatim_api/version.py index 5dd084c7f8..d275f4fc31 100644 --- a/src/nominatim_api/version.py +++ b/src/nominatim_api/version.py @@ -8,10 +8,4 @@ Version information for the Nominatim API. """ -# See also https://github.com/PyCQA/pylint/issues/6006 -# pylint: disable=useless-import-alias,unused-import - -from nominatim_core.version import (NominatimVersion as NominatimVersion, - parse_version as parse_version) - NOMINATIM_API_VERSION = '4.4.99' diff --git a/src/nominatim_core/py.typed b/src/nominatim_core/py.typed deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/nominatim_core/version.py b/src/nominatim_core/version.py deleted file mode 100644 index 91193fa44b..0000000000 --- a/src/nominatim_core/version.py +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later -# -# This file is part of Nominatim. (https://nominatim.org) -# -# Copyright (C) 2024 by the Nominatim developer community. -# For a full list of authors see the git log. -""" -Version information for the Nominatim core package. -""" -from typing import NamedTuple, Optional - -__version__ = '4.4.99' -NOMINATIM_CORE_VERSION = __version__ - -class NominatimVersion(NamedTuple): - """ Version information for Nominatim. We follow semantic versioning. - - Major, minor and patch_level refer to the last released version. - The database patch level tracks important changes between releases - and must always be increased when there is a change to the database or code - that requires a migration. - - When adding a migration on the development branch, raise the patch level - to 99 to make sure that the migration is applied when updating from a - patch release to the next minor version. Patch releases usually shouldn't - have migrations in them. When they are needed, then make sure that the - migration can be reapplied and set the migration version to the appropriate - patch level when cherry-picking the commit with the migration. - """ - - major: int - minor: int - patch_level: int - db_patch_level: Optional[int] - - def __str__(self) -> str: - if self.db_patch_level is None: - return f"{self.major}.{self.minor}.{self.patch_level}" - - return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}" - - def release_version(self) -> str: - """ Return the release version in semantic versioning format. - - The release version does not include the database patch version. - """ - return f"{self.major}.{self.minor}.{self.patch_level}" - - -def parse_version(version: str) -> NominatimVersion: - """ Parse a version string into a version consisting of a tuple of - four ints: major, minor, patch level, database patch level - - This is the reverse operation of `version_str()`. - """ - parts = version.split('.') - return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')]) diff --git a/src/nominatim_db/cli.py b/src/nominatim_db/cli.py index 8a85e0e25b..41684fa1db 100644 --- a/src/nominatim_db/cli.py +++ b/src/nominatim_db/cli.py @@ -16,8 +16,8 @@ import argparse from pathlib import Path -from nominatim_core.config import Configuration -from nominatim_core.errors import UsageError +from .config import Configuration +from .errors import UsageError from .tools.exec_utils import run_php_server from . import clicmd from . import version diff --git a/src/nominatim_db/clicmd/admin.py b/src/nominatim_db/clicmd/admin.py index 7b97557d59..7744595bbd 100644 --- a/src/nominatim_db/clicmd/admin.py +++ b/src/nominatim_db/clicmd/admin.py @@ -11,8 +11,8 @@ import argparse import random -from nominatim_core.errors import UsageError -from nominatim_core.db.connection import connect +from ..errors import UsageError +from ..db.connection import connect from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/api.py b/src/nominatim_db/clicmd/api.py index 25b4bff35a..fac88bdd34 100644 --- a/src/nominatim_db/clicmd/api.py +++ b/src/nominatim_db/clicmd/api.py @@ -14,12 +14,12 @@ import sys from functools import reduce -from nominatim_core.errors import UsageError import nominatim_api as napi import nominatim_api.v1 as api_output from nominatim_api.v1.helpers import zoom_to_rank, deduplicate_results from nominatim_api.v1.format import dispatch as formatting import nominatim_api.logging as loglib +from ..errors import UsageError from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/args.py b/src/nominatim_db/clicmd/args.py index 857cb55efb..6a11b089eb 100644 --- a/src/nominatim_db/clicmd/args.py +++ b/src/nominatim_db/clicmd/args.py @@ -12,9 +12,9 @@ import logging from pathlib import Path -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.typing import Protocol +from ..errors import UsageError +from ..config import Configuration +from ..typing import Protocol LOG = logging.getLogger() diff --git a/src/nominatim_db/clicmd/convert.py b/src/nominatim_db/clicmd/convert.py index 51db848c06..1468b7829f 100644 --- a/src/nominatim_db/clicmd/convert.py +++ b/src/nominatim_db/clicmd/convert.py @@ -12,7 +12,7 @@ import asyncio from pathlib import Path -from nominatim_core.errors import UsageError +from ..errors import UsageError from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/export.py b/src/nominatim_db/clicmd/export.py index 460d27d656..e7fc5c1603 100644 --- a/src/nominatim_db/clicmd/export.py +++ b/src/nominatim_db/clicmd/export.py @@ -14,12 +14,13 @@ import csv import sys -import sqlalchemy as sa - import nominatim_api as napi from nominatim_api.results import create_from_placex_row, ReverseResult, add_result_details from nominatim_api.types import LookupDetails -from nominatim_core.errors import UsageError + +import sqlalchemy as sa + +from ..errors import UsageError from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/freeze.py b/src/nominatim_db/clicmd/freeze.py index 83b84b17fd..27562ccc81 100644 --- a/src/nominatim_db/clicmd/freeze.py +++ b/src/nominatim_db/clicmd/freeze.py @@ -9,7 +9,7 @@ """ import argparse -from nominatim_core.db.connection import connect +from ..db.connection import connect from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/index.py b/src/nominatim_db/clicmd/index.py index 7766a1d10b..87e0fc0333 100644 --- a/src/nominatim_db/clicmd/index.py +++ b/src/nominatim_db/clicmd/index.py @@ -11,8 +11,8 @@ import psutil -from nominatim_core.db import status -from nominatim_core.db.connection import connect +from ..db import status +from ..db.connection import connect from .args import NominatimArgs # Do not repeat documentation of subcommand classes. diff --git a/src/nominatim_db/clicmd/refresh.py b/src/nominatim_db/clicmd/refresh.py index ceead2583f..d5acf54b3a 100644 --- a/src/nominatim_db/clicmd/refresh.py +++ b/src/nominatim_db/clicmd/refresh.py @@ -12,8 +12,8 @@ import logging from pathlib import Path -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect +from ..config import Configuration +from ..db.connection import connect from ..tokenizer.base import AbstractTokenizer from .args import NominatimArgs diff --git a/src/nominatim_db/clicmd/replication.py b/src/nominatim_db/clicmd/replication.py index 6fc4dc302a..f04c730f2d 100644 --- a/src/nominatim_db/clicmd/replication.py +++ b/src/nominatim_db/clicmd/replication.py @@ -14,9 +14,9 @@ import socket import time -from nominatim_core.db import status -from nominatim_core.db.connection import connect -from nominatim_core.errors import UsageError +from ..db import status +from ..db.connection import connect +from ..errors import UsageError from .args import NominatimArgs LOG = logging.getLogger() diff --git a/src/nominatim_db/clicmd/setup.py b/src/nominatim_db/clicmd/setup.py index af586d49f1..f516ba0c0d 100644 --- a/src/nominatim_db/clicmd/setup.py +++ b/src/nominatim_db/clicmd/setup.py @@ -14,10 +14,10 @@ import psutil -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect -from nominatim_core.db import status, properties +from ..errors import UsageError +from ..config import Configuration +from ..db.connection import connect +from ..db import status, properties from ..tokenizer.base import AbstractTokenizer from ..version import NOMINATIM_VERSION from .args import NominatimArgs diff --git a/src/nominatim_db/clicmd/special_phrases.py b/src/nominatim_db/clicmd/special_phrases.py index 1e0f2d764c..660859421b 100644 --- a/src/nominatim_db/clicmd/special_phrases.py +++ b/src/nominatim_db/clicmd/special_phrases.py @@ -11,8 +11,8 @@ import logging from pathlib import Path -from nominatim_core.errors import UsageError -from nominatim_core.db.connection import connect +from ..errors import UsageError +from ..db.connection import connect from ..tools.special_phrases.sp_importer import SPImporter, SpecialPhraseLoader from ..tools.special_phrases.sp_wiki_loader import SPWikiLoader from ..tools.special_phrases.sp_csv_loader import SPCsvLoader diff --git a/src/nominatim_core/config.py b/src/nominatim_db/config.py similarity index 98% rename from src/nominatim_core/config.py rename to src/nominatim_db/config.py index f4c925eb48..c4264f0d68 100644 --- a/src/nominatim_core/config.py +++ b/src/nominatim_db/config.py @@ -17,7 +17,11 @@ import yaml from dotenv import dotenv_values -from psycopg2.extensions import parse_dsn + +try: + from psycopg2.extensions import parse_dsn +except ModuleNotFoundError: + from psycopg.conninfo import conninfo_to_dict as parse_dsn # type: ignore[assignment] from .typing import StrPath from .errors import UsageError diff --git a/src/nominatim_db/data/country_info.py b/src/nominatim_db/data/country_info.py index 35943a50d3..c8002ee702 100644 --- a/src/nominatim_db/data/country_info.py +++ b/src/nominatim_db/data/country_info.py @@ -11,10 +11,10 @@ from pathlib import Path import psycopg2.extras -from nominatim_core.db import utils as db_utils -from nominatim_core.db.connection import connect, Connection -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration +from ..db import utils as db_utils +from ..db.connection import connect, Connection +from ..errors import UsageError +from ..config import Configuration from ..tokenizer.base import AbstractTokenizer def _flatten_name_list(names: Any) -> Dict[str, str]: diff --git a/src/nominatim_db/data/postcode_format.py b/src/nominatim_db/data/postcode_format.py index 670827ee4c..0d04826d94 100644 --- a/src/nominatim_db/data/postcode_format.py +++ b/src/nominatim_db/data/postcode_format.py @@ -11,7 +11,7 @@ from typing import Any, Mapping, Optional, Set, Match import re -from nominatim_core.errors import UsageError +from ..errors import UsageError from . import country_info class CountryPostcodeMatcher: diff --git a/src/nominatim_core/db/__init__.py b/src/nominatim_db/db/__init__.py similarity index 100% rename from src/nominatim_core/db/__init__.py rename to src/nominatim_db/db/__init__.py diff --git a/src/nominatim_core/db/async_connection.py b/src/nominatim_db/db/async_connection.py similarity index 100% rename from src/nominatim_core/db/async_connection.py rename to src/nominatim_db/db/async_connection.py diff --git a/src/nominatim_core/db/connection.py b/src/nominatim_db/db/connection.py similarity index 100% rename from src/nominatim_core/db/connection.py rename to src/nominatim_db/db/connection.py diff --git a/src/nominatim_core/db/properties.py b/src/nominatim_db/db/properties.py similarity index 100% rename from src/nominatim_core/db/properties.py rename to src/nominatim_db/db/properties.py diff --git a/src/nominatim_core/db/sql_preprocessor.py b/src/nominatim_db/db/sql_preprocessor.py similarity index 100% rename from src/nominatim_core/db/sql_preprocessor.py rename to src/nominatim_db/db/sql_preprocessor.py diff --git a/src/nominatim_core/db/status.py b/src/nominatim_db/db/status.py similarity index 100% rename from src/nominatim_core/db/status.py rename to src/nominatim_db/db/status.py diff --git a/src/nominatim_core/db/utils.py b/src/nominatim_db/db/utils.py similarity index 100% rename from src/nominatim_core/db/utils.py rename to src/nominatim_db/db/utils.py diff --git a/src/nominatim_db/errors.py b/src/nominatim_db/errors.py new file mode 100644 index 0000000000..c7331a89f7 --- /dev/null +++ b/src/nominatim_db/errors.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2024 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Custom exception and error classes for Nominatim. +""" + +class UsageError(Exception): + """ An error raised because of bad user input. This error will usually + not cause a stack trace to be printed unless debugging is enabled. + """ diff --git a/src/nominatim_db/indexer/indexer.py b/src/nominatim_db/indexer/indexer.py index 4f9c057554..5a219f6b2e 100644 --- a/src/nominatim_db/indexer/indexer.py +++ b/src/nominatim_db/indexer/indexer.py @@ -13,9 +13,9 @@ import psycopg2.extras -from nominatim_core.typing import DictCursorResults -from nominatim_core.db.async_connection import DBConnection, WorkerPool -from nominatim_core.db.connection import connect, Connection, Cursor +from ..typing import DictCursorResults +from ..db.async_connection import DBConnection, WorkerPool +from ..db.connection import connect, Connection, Cursor from ..tokenizer.base import AbstractTokenizer from .progress import ProgressLogger from . import runners diff --git a/src/nominatim_db/indexer/runners.py b/src/nominatim_db/indexer/runners.py index 4d6a28e4de..7b98e2401b 100644 --- a/src/nominatim_db/indexer/runners.py +++ b/src/nominatim_db/indexer/runners.py @@ -14,8 +14,8 @@ from psycopg2 import sql as pysql import psycopg2.extras -from nominatim_core.typing import Query, DictCursorResult, DictCursorResults, Protocol -from nominatim_core.db.async_connection import DBConnection +from ..typing import Query, DictCursorResult, DictCursorResults, Protocol +from ..db.async_connection import DBConnection from ..data.place_info import PlaceInfo from ..tokenizer.base import AbstractAnalyzer diff --git a/src/nominatim_core/paths.py b/src/nominatim_db/paths.py similarity index 100% rename from src/nominatim_core/paths.py rename to src/nominatim_db/paths.py diff --git a/src/nominatim_db/tokenizer/base.py b/src/nominatim_db/tokenizer/base.py index 0ca7e1d19f..d3aeeacac6 100644 --- a/src/nominatim_db/tokenizer/base.py +++ b/src/nominatim_db/tokenizer/base.py @@ -12,9 +12,9 @@ from typing import List, Tuple, Dict, Any, Optional, Iterable from pathlib import Path -from nominatim_core.typing import Protocol -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection +from ..typing import Protocol +from ..config import Configuration +from ..db.connection import Connection from ..data.place_info import PlaceInfo class AbstractAnalyzer(ABC): diff --git a/src/nominatim_db/tokenizer/factory.py b/src/nominatim_db/tokenizer/factory.py index 1193f49919..b9022d8d01 100644 --- a/src/nominatim_db/tokenizer/factory.py +++ b/src/nominatim_db/tokenizer/factory.py @@ -24,10 +24,10 @@ import importlib from pathlib import Path -from nominatim_core.errors import UsageError -from nominatim_core.db import properties -from nominatim_core.db.connection import connect -from nominatim_core.config import Configuration +from ..errors import UsageError +from ..db import properties +from ..db.connection import connect +from ..config import Configuration from ..tokenizer.base import AbstractTokenizer, TokenizerModule LOG = logging.getLogger() diff --git a/src/nominatim_db/tokenizer/icu_rule_loader.py b/src/nominatim_db/tokenizer/icu_rule_loader.py index 500683e549..0aca2921b9 100644 --- a/src/nominatim_db/tokenizer/icu_rule_loader.py +++ b/src/nominatim_db/tokenizer/icu_rule_loader.py @@ -14,10 +14,10 @@ from icu import Transliterator -from nominatim_core.config import flatten_config_list, Configuration -from nominatim_core.db.properties import set_property, get_property -from nominatim_core.db.connection import Connection -from nominatim_core.errors import UsageError +from ..config import flatten_config_list, Configuration +from ..db.properties import set_property, get_property +from ..db.connection import Connection +from ..errors import UsageError from .place_sanitizer import PlaceSanitizer from .icu_token_analysis import ICUTokenAnalysis from .token_analysis.base import AnalysisModule, Analyzer diff --git a/src/nominatim_db/tokenizer/icu_tokenizer.py b/src/nominatim_db/tokenizer/icu_tokenizer.py index af03bd3981..22e2d04829 100644 --- a/src/nominatim_db/tokenizer/icu_tokenizer.py +++ b/src/nominatim_db/tokenizer/icu_tokenizer.py @@ -16,10 +16,10 @@ from pathlib import Path from textwrap import dedent -from nominatim_core.db.connection import connect, Connection, Cursor -from nominatim_core.config import Configuration -from nominatim_core.db.utils import CopyBuffer -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..db.connection import connect, Connection, Cursor +from ..config import Configuration +from ..db.utils import CopyBuffer +from ..db.sql_preprocessor import SQLPreprocessor from ..data.place_info import PlaceInfo from ..data.place_name import PlaceName from .icu_rule_loader import ICURuleLoader diff --git a/src/nominatim_db/tokenizer/legacy_tokenizer.py b/src/nominatim_db/tokenizer/legacy_tokenizer.py index 5c6ba7434d..136a733159 100644 --- a/src/nominatim_db/tokenizer/legacy_tokenizer.py +++ b/src/nominatim_db/tokenizer/legacy_tokenizer.py @@ -20,12 +20,12 @@ import psycopg2 import psycopg2.extras -from nominatim_core.errors import UsageError -from nominatim_core.db.connection import connect, Connection -from nominatim_core.config import Configuration -from nominatim_core.db import properties -from nominatim_core.db import utils as db_utils -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..errors import UsageError +from ..db.connection import connect, Connection +from ..config import Configuration +from ..db import properties +from ..db import utils as db_utils +from ..db.sql_preprocessor import SQLPreprocessor from ..data.place_info import PlaceInfo from .base import AbstractAnalyzer, AbstractTokenizer diff --git a/src/nominatim_db/tokenizer/place_sanitizer.py b/src/nominatim_db/tokenizer/place_sanitizer.py index 546d0da33c..68322f9fd5 100644 --- a/src/nominatim_db/tokenizer/place_sanitizer.py +++ b/src/nominatim_db/tokenizer/place_sanitizer.py @@ -10,8 +10,8 @@ """ from typing import Optional, List, Mapping, Sequence, Callable, Any, Tuple -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration +from ..errors import UsageError +from ..config import Configuration from .sanitizers.config import SanitizerConfig from .sanitizers.base import SanitizerHandler, ProcessInfo from ..data.place_name import PlaceName diff --git a/src/nominatim_db/tokenizer/sanitizers/base.py b/src/nominatim_db/tokenizer/sanitizers/base.py index a28f86b503..2dbc448255 100644 --- a/src/nominatim_db/tokenizer/sanitizers/base.py +++ b/src/nominatim_db/tokenizer/sanitizers/base.py @@ -9,7 +9,7 @@ """ from typing import Optional, List, Mapping, Callable -from nominatim_core.typing import Protocol, Final +from ...typing import Protocol, Final from ...data.place_info import PlaceInfo from ...data.place_name import PlaceName from .config import SanitizerConfig diff --git a/src/nominatim_db/tokenizer/sanitizers/config.py b/src/nominatim_db/tokenizer/sanitizers/config.py index 1ce1a1b72a..034d0791ed 100644 --- a/src/nominatim_db/tokenizer/sanitizers/config.py +++ b/src/nominatim_db/tokenizer/sanitizers/config.py @@ -11,7 +11,7 @@ from collections import UserDict import re -from nominatim_core.errors import UsageError +from ...errors import UsageError # working around missing generics in Python < 3.8 # See https://github.com/python/typing/issues/60#issuecomment-869757075 diff --git a/src/nominatim_db/tokenizer/token_analysis/base.py b/src/nominatim_db/tokenizer/token_analysis/base.py index fc6734911f..9435edb3b5 100644 --- a/src/nominatim_db/tokenizer/token_analysis/base.py +++ b/src/nominatim_db/tokenizer/token_analysis/base.py @@ -9,7 +9,7 @@ """ from typing import Mapping, List, Any -from nominatim_core.typing import Protocol +from ...typing import Protocol from ...data.place_name import PlaceName class Analyzer(Protocol): diff --git a/src/nominatim_db/tokenizer/token_analysis/config_variants.py b/src/nominatim_db/tokenizer/token_analysis/config_variants.py index b455a8a5df..aff360afa6 100644 --- a/src/nominatim_db/tokenizer/token_analysis/config_variants.py +++ b/src/nominatim_db/tokenizer/token_analysis/config_variants.py @@ -12,8 +12,8 @@ import itertools import re -from nominatim_core.config import flatten_config_list -from nominatim_core.errors import UsageError +from ...config import flatten_config_list +from ...errors import UsageError class ICUVariant(NamedTuple): """ A single replacement rule for variant creation. diff --git a/src/nominatim_db/tokenizer/token_analysis/generic.py b/src/nominatim_db/tokenizer/token_analysis/generic.py index cd649e62f5..30f1944e56 100644 --- a/src/nominatim_db/tokenizer/token_analysis/generic.py +++ b/src/nominatim_db/tokenizer/token_analysis/generic.py @@ -12,7 +12,7 @@ import datrie -from nominatim_core.errors import UsageError +from ...errors import UsageError from ...data.place_name import PlaceName from .config_variants import get_variant_config from .generic_mutation import MutationVariantGenerator diff --git a/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py b/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py index 03cc63db0a..be70b49dcd 100644 --- a/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py +++ b/src/nominatim_db/tokenizer/token_analysis/generic_mutation.py @@ -12,7 +12,7 @@ import logging import re -from nominatim_core.errors import UsageError +from ...errors import UsageError LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/add_osm_data.py b/src/nominatim_db/tools/add_osm_data.py index 44ff21975d..f9e2e6486f 100644 --- a/src/nominatim_db/tools/add_osm_data.py +++ b/src/nominatim_db/tools/add_osm_data.py @@ -12,8 +12,8 @@ import logging import urllib -from nominatim_core.db.connection import connect -from nominatim_core.utils.url_utils import get_url +from ..db.connection import connect +from ..utils.url_utils import get_url from .exec_utils import run_osm2pgsql LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/admin.py b/src/nominatim_db/tools/admin.py index 7cc0f04794..cea2ad664a 100644 --- a/src/nominatim_db/tools/admin.py +++ b/src/nominatim_db/tools/admin.py @@ -13,10 +13,10 @@ from psycopg2.extras import Json, register_hstore from psycopg2 import DataError -from nominatim_core.typing import DictCursorResult -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect, Cursor -from nominatim_core.errors import UsageError +from ..typing import DictCursorResult +from ..config import Configuration +from ..db.connection import connect, Cursor +from ..errors import UsageError from ..tokenizer import factory as tokenizer_factory from ..data.place_info import PlaceInfo diff --git a/src/nominatim_db/tools/check_database.py b/src/nominatim_db/tools/check_database.py index d2659877c5..ef28a0e5a6 100644 --- a/src/nominatim_db/tools/check_database.py +++ b/src/nominatim_db/tools/check_database.py @@ -11,10 +11,10 @@ from enum import Enum from textwrap import dedent -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect, Connection -from nominatim_core.db import properties -from nominatim_core.errors import UsageError +from ..config import Configuration +from ..db.connection import connect, Connection +from ..db import properties +from ..errors import UsageError from ..tokenizer import factory as tokenizer_factory from . import freeze from ..version import NOMINATIM_VERSION, parse_version diff --git a/src/nominatim_db/tools/collect_os_info.py b/src/nominatim_db/tools/collect_os_info.py index 5096ce69eb..e1f8b16637 100644 --- a/src/nominatim_db/tools/collect_os_info.py +++ b/src/nominatim_db/tools/collect_os_info.py @@ -17,8 +17,8 @@ import psutil from psycopg2.extensions import make_dsn, parse_dsn -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect +from ..config import Configuration +from ..db.connection import connect from ..version import NOMINATIM_VERSION diff --git a/src/nominatim_db/tools/convert_sqlite.py b/src/nominatim_db/tools/convert_sqlite.py index 47ab63b718..2377abc0b2 100644 --- a/src/nominatim_db/tools/convert_sqlite.py +++ b/src/nominatim_db/tools/convert_sqlite.py @@ -16,8 +16,8 @@ import nominatim_api as napi from nominatim_api.search.query_analyzer_factory import make_query_analyzer -from nominatim_core.typing import SaSelect, SaRow -from nominatim_core.db.sqlalchemy_types import Geometry, IntArray +from nominatim_api.typing import SaSelect, SaRow +from nominatim_api.sql.sqlalchemy_types import Geometry, IntArray LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/database_import.py b/src/nominatim_db/tools/database_import.py index 84f2f32542..d07febc8a3 100644 --- a/src/nominatim_db/tools/database_import.py +++ b/src/nominatim_db/tools/database_import.py @@ -17,11 +17,11 @@ import psutil from psycopg2 import sql as pysql -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect, get_pg_env, Connection -from nominatim_core.db.async_connection import DBConnection -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..errors import UsageError +from ..config import Configuration +from ..db.connection import connect, get_pg_env, Connection +from ..db.async_connection import DBConnection +from ..db.sql_preprocessor import SQLPreprocessor from .exec_utils import run_osm2pgsql from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION diff --git a/src/nominatim_db/tools/exec_utils.py b/src/nominatim_db/tools/exec_utils.py index 467347f871..406e2511e8 100644 --- a/src/nominatim_db/tools/exec_utils.py +++ b/src/nominatim_db/tools/exec_utils.py @@ -13,8 +13,8 @@ import subprocess import shutil -from nominatim_core.typing import StrPath -from nominatim_core.db.connection import get_pg_env +from ..typing import StrPath +from ..db.connection import get_pg_env LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/freeze.py b/src/nominatim_db/tools/freeze.py index 2023bb8ac4..bd52ba9a5a 100644 --- a/src/nominatim_db/tools/freeze.py +++ b/src/nominatim_db/tools/freeze.py @@ -12,7 +12,7 @@ from psycopg2 import sql as pysql -from nominatim_core.db.connection import Connection +from ..db.connection import Connection UPDATE_TABLES = [ 'address_levels', diff --git a/src/nominatim_db/tools/migration.py b/src/nominatim_db/tools/migration.py index 0712f187ff..e6803c7dea 100644 --- a/src/nominatim_db/tools/migration.py +++ b/src/nominatim_db/tools/migration.py @@ -12,10 +12,10 @@ from psycopg2 import sql as pysql -from nominatim_core.errors import UsageError -from nominatim_core.config import Configuration -from nominatim_core.db import properties -from nominatim_core.db.connection import connect, Connection +from ..errors import UsageError +from ..config import Configuration +from ..db import properties +from ..db.connection import connect, Connection from ..version import NominatimVersion, NOMINATIM_VERSION, parse_version from ..tokenizer import factory as tokenizer_factory from . import refresh diff --git a/src/nominatim_db/tools/postcodes.py b/src/nominatim_db/tools/postcodes.py index 772651b37d..8dc5bdbdb4 100644 --- a/src/nominatim_db/tools/postcodes.py +++ b/src/nominatim_db/tools/postcodes.py @@ -18,8 +18,8 @@ from psycopg2 import sql as pysql -from nominatim_core.db.connection import connect, Connection -from nominatim_core.utils.centroid import PointsCentroid +from ..db.connection import connect, Connection +from ..utils.centroid import PointsCentroid from ..data.postcode_format import PostcodeFormatter, CountryPostcodeMatcher from ..tokenizer.base import AbstractAnalyzer, AbstractTokenizer diff --git a/src/nominatim_db/tools/refresh.py b/src/nominatim_db/tools/refresh.py index 966080976f..6946a41a44 100644 --- a/src/nominatim_db/tools/refresh.py +++ b/src/nominatim_db/tools/refresh.py @@ -16,10 +16,10 @@ from psycopg2 import sql as pysql -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection, connect -from nominatim_core.db.utils import execute_file, CopyBuffer -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from ..config import Configuration +from ..db.connection import Connection, connect +from ..db.utils import execute_file, CopyBuffer +from ..db.sql_preprocessor import SQLPreprocessor from ..version import NOMINATIM_VERSION LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/replication.py b/src/nominatim_db/tools/replication.py index f9421bb88e..bf1189df03 100644 --- a/src/nominatim_db/tools/replication.py +++ b/src/nominatim_db/tools/replication.py @@ -18,9 +18,9 @@ import requests -from nominatim_core.errors import UsageError -from nominatim_core.db import status -from nominatim_core.db.connection import Connection, connect +from ..errors import UsageError +from ..db import status +from ..db.connection import Connection, connect from .exec_utils import run_osm2pgsql try: diff --git a/src/nominatim_db/tools/special_phrases/sp_csv_loader.py b/src/nominatim_db/tools/special_phrases/sp_csv_loader.py index 9f472e682f..db4806cdd0 100644 --- a/src/nominatim_db/tools/special_phrases/sp_csv_loader.py +++ b/src/nominatim_db/tools/special_phrases/sp_csv_loader.py @@ -13,7 +13,7 @@ import csv import os -from nominatim_core.errors import UsageError +from ...errors import UsageError from .special_phrase import SpecialPhrase class SPCsvLoader: diff --git a/src/nominatim_db/tools/special_phrases/sp_importer.py b/src/nominatim_db/tools/special_phrases/sp_importer.py index 9749db3e69..1bdcdaf133 100644 --- a/src/nominatim_db/tools/special_phrases/sp_importer.py +++ b/src/nominatim_db/tools/special_phrases/sp_importer.py @@ -19,9 +19,9 @@ from psycopg2.sql import Identifier, SQL -from nominatim_core.typing import Protocol -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection +from ...typing import Protocol +from ...config import Configuration +from ...db.connection import Connection from .importer_statistics import SpecialPhrasesImporterStatistics from .special_phrase import SpecialPhrase from ...tokenizer.base import AbstractTokenizer diff --git a/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py b/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py index c7e3833606..0fe7c0aa4f 100644 --- a/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py +++ b/src/nominatim_db/tools/special_phrases/sp_wiki_loader.py @@ -11,8 +11,8 @@ import re import logging -from nominatim_core.config import Configuration -from nominatim_core.utils.url_utils import get_url +from ...config import Configuration +from ...utils.url_utils import get_url from .special_phrase import SpecialPhrase LOG = logging.getLogger() diff --git a/src/nominatim_db/tools/tiger_data.py b/src/nominatim_db/tools/tiger_data.py index 6030a38aef..7c52b7102a 100644 --- a/src/nominatim_db/tools/tiger_data.py +++ b/src/nominatim_db/tools/tiger_data.py @@ -16,11 +16,11 @@ from psycopg2.extras import Json -from nominatim_core.config import Configuration -from nominatim_core.db.connection import connect -from nominatim_core.db.async_connection import WorkerPool -from nominatim_core.db.sql_preprocessor import SQLPreprocessor -from nominatim_core.errors import UsageError +from ..config import Configuration +from ..db.connection import connect +from ..db.async_connection import WorkerPool +from ..db.sql_preprocessor import SQLPreprocessor +from ..errors import UsageError from ..data.place_info import PlaceInfo from ..tokenizer.base import AbstractAnalyzer, AbstractTokenizer from . import freeze diff --git a/src/nominatim_core/typing.py b/src/nominatim_db/typing.py similarity index 65% rename from src/nominatim_core/typing.py rename to src/nominatim_db/typing.py index 1f4f1fd7d2..f1abee8280 100644 --- a/src/nominatim_core/typing.py +++ b/src/nominatim_db/typing.py @@ -50,26 +50,3 @@ def __getitem__(self, x: Union[int, str]) -> Any: ... Protocol = object Final = 'Final' TypedDict = dict - - -# SQLAlchemy introduced generic types in version 2.0 making typing -# incompatible with older versions. Add wrappers here so we don't have -# to litter the code with bare-string types. - -if TYPE_CHECKING: - import sqlalchemy as sa - from typing_extensions import (TypeAlias as TypeAlias) -else: - TypeAlias = str - -SaLambdaSelect: TypeAlias = 'Union[sa.Select[Any], sa.StatementLambdaElement]' -SaSelect: TypeAlias = 'sa.Select[Any]' -SaScalarSelect: TypeAlias = 'sa.ScalarSelect[Any]' -SaRow: TypeAlias = 'sa.Row[Any]' -SaColumn: TypeAlias = 'sa.ColumnElement[Any]' -SaExpression: TypeAlias = 'sa.ColumnElement[bool]' -SaLabel: TypeAlias = 'sa.Label[Any]' -SaFromClause: TypeAlias = 'sa.FromClause' -SaSelectable: TypeAlias = 'sa.Selectable' -SaBind: TypeAlias = 'sa.BindParameter[Any]' -SaDialect: TypeAlias = 'sa.Dialect' diff --git a/src/nominatim_core/utils/__init__.py b/src/nominatim_db/utils/__init__.py similarity index 100% rename from src/nominatim_core/utils/__init__.py rename to src/nominatim_db/utils/__init__.py diff --git a/src/nominatim_core/utils/centroid.py b/src/nominatim_db/utils/centroid.py similarity index 100% rename from src/nominatim_core/utils/centroid.py rename to src/nominatim_db/utils/centroid.py diff --git a/src/nominatim_core/utils/url_utils.py b/src/nominatim_db/utils/url_utils.py similarity index 87% rename from src/nominatim_core/utils/url_utils.py rename to src/nominatim_db/utils/url_utils.py index d7689187a1..eb56f72ed3 100644 --- a/src/nominatim_core/utils/url_utils.py +++ b/src/nominatim_db/utils/url_utils.py @@ -11,7 +11,7 @@ import logging import urllib.request as urlrequest -from ..version import NOMINATIM_CORE_VERSION +from ..version import NOMINATIM_VERSION LOG = logging.getLogger() @@ -20,7 +20,7 @@ def get_url(url: str) -> str: This version makes sure that an appropriate user agent is sent. """ - headers = {"User-Agent": f"Nominatim/{NOMINATIM_CORE_VERSION!s}"} + headers = {"User-Agent": f"Nominatim/{NOMINATIM_VERSION!s}"} try: request = urlrequest.Request(url, headers=headers) diff --git a/src/nominatim_db/version.py b/src/nominatim_db/version.py index 34e3b432e3..70e1ac14ac 100644 --- a/src/nominatim_db/version.py +++ b/src/nominatim_db/version.py @@ -7,13 +7,55 @@ """ Version information for Nominatim. """ -from typing import Optional +from typing import NamedTuple, Optional # See also https://github.com/PyCQA/pylint/issues/6006 # pylint: disable=useless-import-alias,unused-import -from nominatim_core.version import (NominatimVersion as NominatimVersion, - parse_version as parse_version) +class NominatimVersion(NamedTuple): + """ Version information for Nominatim. We follow semantic versioning. + + Major, minor and patch_level refer to the last released version. + The database patch level tracks important changes between releases + and must always be increased when there is a change to the database or code + that requires a migration. + + When adding a migration on the development branch, raise the patch level + to 99 to make sure that the migration is applied when updating from a + patch release to the next minor version. Patch releases usually shouldn't + have migrations in them. When they are needed, then make sure that the + migration can be reapplied and set the migration version to the appropriate + patch level when cherry-picking the commit with the migration. + """ + + major: int + minor: int + patch_level: int + db_patch_level: Optional[int] + + def __str__(self) -> str: + if self.db_patch_level is None: + return f"{self.major}.{self.minor}.{self.patch_level}" + + return f"{self.major}.{self.minor}.{self.patch_level}-{self.db_patch_level}" + + def release_version(self) -> str: + """ Return the release version in semantic versioning format. + + The release version does not include the database patch version. + """ + return f"{self.major}.{self.minor}.{self.patch_level}" + + +def parse_version(version: str) -> NominatimVersion: + """ Parse a version string into a version consisting of a tuple of + four ints: major, minor, patch level, database patch level + + This is the reverse operation of `version_str()`. + """ + parts = version.split('.') + return NominatimVersion(*[int(x) for x in parts[:2] + parts[2].split('-')]) + NOMINATIM_VERSION = parse_version('4.4.99-1') diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 46559720d3..dfbbee2874 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -15,8 +15,8 @@ sys.path.insert(1, str((Path(__file__) / '..' / '..' / '..' / '..'/ 'src').resolve())) from nominatim_db import cli -from nominatim_core.config import Configuration -from nominatim_core.db.connection import Connection +from nominatim_db.config import Configuration +from nominatim_db.db.connection import Connection from nominatim_db.tools import refresh from nominatim_db.tokenizer import factory as tokenizer_factory from steps.utils import run_script diff --git a/test/python/api/conftest.py b/test/python/api/conftest.py index 08b7422c9c..a902e2640a 100644 --- a/test/python/api/conftest.py +++ b/test/python/api/conftest.py @@ -15,7 +15,7 @@ import sqlalchemy as sa import nominatim_api as napi -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.db.sql_preprocessor import SQLPreprocessor from nominatim_api.search.query_analyzer_factory import make_query_analyzer from nominatim_db.tools import convert_sqlite import nominatim_api.logging as loglib diff --git a/test/python/api/fake_adaptor.py b/test/python/api/fake_adaptor.py index 50fbbbbea7..9caa922665 100644 --- a/test/python/api/fake_adaptor.py +++ b/test/python/api/fake_adaptor.py @@ -10,7 +10,7 @@ from collections import namedtuple import nominatim_api.v1.server_glue as glue -from nominatim_core.config import Configuration +from nominatim_api.config import Configuration class FakeError(BaseException): diff --git a/test/python/api/test_api_status.py b/test/python/api/test_api_status.py index 2b211611a7..5412ca6e33 100644 --- a/test/python/api/test_api_status.py +++ b/test/python/api/test_api_status.py @@ -39,7 +39,7 @@ def test_status_full(apiobj, frontend): assert result.status == 0 assert result.message == 'OK' assert result.software_version == NOMINATIM_API_VERSION - assert result.database_version == NominatimVersion(99, 5, 4, 2) + assert result.database_version == '99.5.4-2' assert result.data_updated == import_date diff --git a/test/python/api/test_api_types.py b/test/python/api/test_api_types.py index 2faba33dac..fbb9b682f4 100644 --- a/test/python/api/test_api_types.py +++ b/test/python/api/test_api_types.py @@ -9,7 +9,7 @@ """ import pytest -from nominatim_core.errors import UsageError +from nominatim_api.errors import UsageError import nominatim_api.types as typ def test_no_params_defaults(): diff --git a/test/python/cli/test_cmd_import.py b/test/python/cli/test_cmd_import.py index 9c2400c5c2..85235e1e2e 100644 --- a/test/python/cli/test_cmd_import.py +++ b/test/python/cli/test_cmd_import.py @@ -14,7 +14,7 @@ import nominatim_db.tools.refresh import nominatim_db.tools.postcodes import nominatim_db.indexer.indexer -import nominatim_core.db.properties +import nominatim_db.db.properties class TestCliImportWithDb: @@ -81,7 +81,7 @@ def test_import_continue_load_data(self, mock_func_factory): mock_func_factory(nominatim_db.tools.postcodes, 'update_postcodes'), mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim_db.tools.refresh, 'setup_website'), - mock_func_factory(nominatim_core.db.properties, 'set_property') + mock_func_factory(nominatim_db.db.properties, 'set_property') ] assert self.call_nominatim('import', '--continue', 'load-data') == 0 @@ -98,7 +98,7 @@ def test_import_continue_indexing(self, mock_func_factory, placex_table, mock_func_factory(nominatim_db.data.country_info, 'create_country_names'), mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim_db.tools.refresh, 'setup_website'), - mock_func_factory(nominatim_core.db.properties, 'set_property') + mock_func_factory(nominatim_db.db.properties, 'set_property') ] assert self.call_nominatim('import', '--continue', 'indexing') == 0 @@ -115,7 +115,7 @@ def test_import_continue_postprocess(self, mock_func_factory): mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'), mock_func_factory(nominatim_db.data.country_info, 'create_country_names'), mock_func_factory(nominatim_db.tools.refresh, 'setup_website'), - mock_func_factory(nominatim_core.db.properties, 'set_property') + mock_func_factory(nominatim_db.db.properties, 'set_property') ] assert self.call_nominatim('import', '--continue', 'db-postprocess') == 0 diff --git a/test/python/cli/test_cmd_replication.py b/test/python/cli/test_cmd_replication.py index ff900e58bb..8c1e8ea644 100644 --- a/test/python/cli/test_cmd_replication.py +++ b/test/python/cli/test_cmd_replication.py @@ -16,7 +16,7 @@ import nominatim_db.indexer.indexer import nominatim_db.tools.replication import nominatim_db.tools.refresh -from nominatim_core.db import status +from nominatim_db.db import status @pytest.fixture def tokenizer_mock(monkeypatch): diff --git a/test/python/config/test_config.py b/test/python/config/test_config.py index bb649d2328..5c9393ecfd 100644 --- a/test/python/config/test_config.py +++ b/test/python/config/test_config.py @@ -10,8 +10,8 @@ from pathlib import Path import pytest -from nominatim_core.config import Configuration, flatten_config_list -from nominatim_core.errors import UsageError +from nominatim_db.config import Configuration, flatten_config_list +from nominatim_db.errors import UsageError @pytest.fixture def make_config(): diff --git a/test/python/config/test_config_load_module.py b/test/python/config/test_config_load_module.py index 06bfff065f..c29121802c 100644 --- a/test/python/config/test_config_load_module.py +++ b/test/python/config/test_config_load_module.py @@ -12,7 +12,7 @@ import pytest -from nominatim_core.config import Configuration +from nominatim_db.config import Configuration @pytest.fixture def test_config(src_dir, tmp_path): diff --git a/test/python/conftest.py b/test/python/conftest.py index c9e8f0487f..d301c9736e 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -15,9 +15,9 @@ SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve() sys.path.insert(0, str(SRC_DIR / 'src')) -from nominatim_core.config import Configuration -from nominatim_core.db import connection -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.config import Configuration +from nominatim_db.db import connection +from nominatim_db.db.sql_preprocessor import SQLPreprocessor import nominatim_db.tokenizer.factory import dummy_tokenizer diff --git a/test/python/db/test_async_connection.py b/test/python/db/test_async_connection.py index 019f53c15d..fff695e53e 100644 --- a/test/python/db/test_async_connection.py +++ b/test/python/db/test_async_connection.py @@ -13,7 +13,7 @@ import pytest import psycopg2 -from nominatim_core.db.async_connection import DBConnection, DeadlockHandler +from nominatim_db.db.async_connection import DBConnection, DeadlockHandler @pytest.fixture diff --git a/test/python/db/test_connection.py b/test/python/db/test_connection.py index 7ecf3bb4e2..8b4cc62f0c 100644 --- a/test/python/db/test_connection.py +++ b/test/python/db/test_connection.py @@ -10,7 +10,7 @@ import pytest import psycopg2 -from nominatim_core.db.connection import connect, get_pg_env +from nominatim_db.db.connection import connect, get_pg_env @pytest.fixture def db(dsn): diff --git a/test/python/db/test_properties.py b/test/python/db/test_properties.py index aca1b1f8a9..e55bb97346 100644 --- a/test/python/db/test_properties.py +++ b/test/python/db/test_properties.py @@ -9,7 +9,7 @@ """ import pytest -from nominatim_core.db import properties +from nominatim_db.db import properties @pytest.fixture def property_factory(property_table, temp_db_cursor): diff --git a/test/python/db/test_sql_preprocessor.py b/test/python/db/test_sql_preprocessor.py index 1ef6bc88ce..114c5244e1 100644 --- a/test/python/db/test_sql_preprocessor.py +++ b/test/python/db/test_sql_preprocessor.py @@ -9,7 +9,7 @@ """ import pytest -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.db.sql_preprocessor import SQLPreprocessor @pytest.fixture def sql_factory(tmp_path): diff --git a/test/python/db/test_status.py b/test/python/db/test_status.py index 67cd22e91d..77135a8c7f 100644 --- a/test/python/db/test_status.py +++ b/test/python/db/test_status.py @@ -11,8 +11,8 @@ import pytest -import nominatim_core.db.status -from nominatim_core.errors import UsageError +import nominatim_db.db.status +from nominatim_db.errors import UsageError OSM_NODE_DATA = """\ @@ -22,7 +22,7 @@ """ def iso_date(date): - return dt.datetime.strptime(date, nominatim_core.db.status.ISODATE_FORMAT)\ + return dt.datetime.strptime(date, nominatim_db.db.status.ISODATE_FORMAT)\ .replace(tzinfo=dt.timezone.utc) @@ -36,7 +36,7 @@ def test_compute_database_date_from_osm2pgsql(table_factory, temp_db_conn, offli table_factory('osm2pgsql_properties', 'property TEXT, value TEXT', content=(('current_timestamp', '2024-01-03T23:45:54Z'), )) - date = nominatim_core.db.status.compute_database_date(temp_db_conn, offline=offline) + date = nominatim_db.db.status.compute_database_date(temp_db_conn, offline=offline) assert date == iso_date('2024-01-03T23:45:54') @@ -44,12 +44,12 @@ def test_compute_database_date_from_osm2pgsql_nodata(table_factory, temp_db_conn table_factory('osm2pgsql_properties', 'property TEXT, value TEXT') with pytest.raises(UsageError, match='Cannot determine database date from data in offline mode'): - nominatim_core.db.status.compute_database_date(temp_db_conn, offline=True) + nominatim_db.db.status.compute_database_date(temp_db_conn, offline=True) def test_compute_database_date_place_empty(place_table, temp_db_conn): with pytest.raises(UsageError): - nominatim_core.db.status.compute_database_date(temp_db_conn) + nominatim_db.db.status.compute_database_date(temp_db_conn) def test_compute_database_date_valid(monkeypatch, place_row, temp_db_conn): @@ -60,9 +60,9 @@ def mock_url(url): requested_url.append(url) return OSM_NODE_DATA - monkeypatch.setattr(nominatim_core.db.status, "get_url", mock_url) + monkeypatch.setattr(nominatim_db.db.status, "get_url", mock_url) - date = nominatim_core.db.status.compute_database_date(temp_db_conn) + date = nominatim_db.db.status.compute_database_date(temp_db_conn) assert requested_url == ['https://www.openstreetmap.org/api/0.6/node/45673/1'] assert date == iso_date('2006-01-27T22:09:10') @@ -76,15 +76,15 @@ def mock_url(url): requested_url.append(url) return ' ' '; diff --git a/test/python/tokenizer/token_analysis/test_generic.py b/test/python/tokenizer/token_analysis/test_generic.py index 277758e1ff..191f551f86 100644 --- a/test/python/tokenizer/token_analysis/test_generic.py +++ b/test/python/tokenizer/token_analysis/test_generic.py @@ -12,7 +12,7 @@ from icu import Transliterator import nominatim_db.tokenizer.token_analysis.generic as module -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError DEFAULT_NORMALIZATION = """ :: NFD (); '🜳' > ' '; diff --git a/test/python/tokenizer/token_analysis/test_generic_mutation.py b/test/python/tokenizer/token_analysis/test_generic_mutation.py index ea813bdc7f..7d0db92582 100644 --- a/test/python/tokenizer/token_analysis/test_generic_mutation.py +++ b/test/python/tokenizer/token_analysis/test_generic_mutation.py @@ -12,7 +12,7 @@ from icu import Transliterator import nominatim_db.tokenizer.token_analysis.generic as module -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError DEFAULT_NORMALIZATION = """ '🜳' > ' '; [[:Nonspacing Mark:] [:Cf:]] >; diff --git a/test/python/tools/test_admin.py b/test/python/tools/test_admin.py index a59f3d9da1..1e1f0e2947 100644 --- a/test/python/tools/test_admin.py +++ b/test/python/tools/test_admin.py @@ -9,10 +9,10 @@ """ import pytest -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError from nominatim_db.tools import admin from nominatim_db.tokenizer import factory -from nominatim_core.db.sql_preprocessor import SQLPreprocessor +from nominatim_db.db.sql_preprocessor import SQLPreprocessor @pytest.fixture(autouse=True) def create_placex_table(project_env, tokenizer_mock, temp_db_cursor, placex_table): diff --git a/test/python/tools/test_database_import.py b/test/python/tools/test_database_import.py index a95f3d9804..548ec800b6 100644 --- a/test/python/tools/test_database_import.py +++ b/test/python/tools/test_database_import.py @@ -14,7 +14,7 @@ import psycopg2 from nominatim_db.tools import database_import -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError class TestDatabaseSetup: DBNAME = 'test_nominatim_python_unittest' diff --git a/test/python/tools/test_exec_utils.py b/test/python/tools/test_exec_utils.py index 50bec7e6ea..666ef0b8b6 100644 --- a/test/python/tools/test_exec_utils.py +++ b/test/python/tools/test_exec_utils.py @@ -12,7 +12,7 @@ import pytest -from nominatim_core.config import Configuration +from nominatim_db.config import Configuration import nominatim_db.tools.exec_utils as exec_utils def test_run_osm2pgsql(osm2pgsql_options): diff --git a/test/python/tools/test_import_special_phrases.py b/test/python/tools/test_import_special_phrases.py index 91c9fc1536..64eb7b1856 100644 --- a/test/python/tools/test_import_special_phrases.py +++ b/test/python/tools/test_import_special_phrases.py @@ -13,7 +13,7 @@ from nominatim_db.tools.special_phrases.sp_importer import SPImporter from nominatim_db.tools.special_phrases.sp_wiki_loader import SPWikiLoader from nominatim_db.tools.special_phrases.special_phrase import SpecialPhrase -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError from cursor import CursorForTesting diff --git a/test/python/tools/test_migration.py b/test/python/tools/test_migration.py index d33ff42236..3a849adbf9 100644 --- a/test/python/tools/test_migration.py +++ b/test/python/tools/test_migration.py @@ -11,7 +11,7 @@ import psycopg2.extras from nominatim_db.tools import migration -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError import nominatim_db.version from mock_legacy_word_table import MockLegacyWordTable diff --git a/test/python/tools/test_replication.py b/test/python/tools/test_replication.py index 4cd53ae3aa..392ea0750b 100644 --- a/test/python/tools/test_replication.py +++ b/test/python/tools/test_replication.py @@ -14,8 +14,8 @@ from osmium.replication.server import OsmosisState import nominatim_db.tools.replication -import nominatim_core.db.status as status -from nominatim_core.errors import UsageError +import nominatim_db.db.status as status +from nominatim_db.errors import UsageError OSM_NODE_DATA = """\ diff --git a/test/python/tools/test_sp_csv_loader.py b/test/python/tools/test_sp_csv_loader.py index e6185fd61a..9d0ad9cc8e 100644 --- a/test/python/tools/test_sp_csv_loader.py +++ b/test/python/tools/test_sp_csv_loader.py @@ -9,7 +9,7 @@ """ import pytest -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError from nominatim_db.tools.special_phrases.sp_csv_loader import SPCsvLoader from nominatim_db.tools.special_phrases.special_phrase import SpecialPhrase diff --git a/test/python/tools/test_tiger_data.py b/test/python/tools/test_tiger_data.py index 396fe069f0..7ef6a1e633 100644 --- a/test/python/tools/test_tiger_data.py +++ b/test/python/tools/test_tiger_data.py @@ -13,7 +13,7 @@ import pytest from nominatim_db.tools import tiger_data, freeze -from nominatim_core.errors import UsageError +from nominatim_db.errors import UsageError class MockTigerTable: diff --git a/test/python/utils/test_centroid.py b/test/python/utils/test_centroid.py index d91edae2d4..bac0edb3c4 100644 --- a/test/python/utils/test_centroid.py +++ b/test/python/utils/test_centroid.py @@ -9,7 +9,7 @@ """ import pytest -from nominatim_core.utils.centroid import PointsCentroid +from nominatim_db.utils.centroid import PointsCentroid def test_empty_set(): c = PointsCentroid() diff --git a/test/python/utils/test_json_writer.py b/test/python/utils/test_json_writer.py index ff542f045e..53e3f4d307 100644 --- a/test/python/utils/test_json_writer.py +++ b/test/python/utils/test_json_writer.py @@ -11,7 +11,7 @@ import pytest -from nominatim_core.utils.json_writer import JsonWriter +from nominatim_api.utils.json_writer import JsonWriter @pytest.mark.parametrize("inval,outstr", [(None, 'null'), (True, 'true'), (False, 'false'), diff --git a/vagrant/Install-on-Ubuntu-24.sh b/vagrant/Install-on-Ubuntu-24.sh index 3411c2ba05..05dc38a4cf 100644 --- a/vagrant/Install-on-Ubuntu-24.sh +++ b/vagrant/Install-on-Ubuntu-24.sh @@ -119,7 +119,7 @@ fi #DOCS: # Now install Nominatim using pip: cd $USERHOME/Nominatim - $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-{core,db} + $USERHOME/nominatim-venv/bin/pip install packaging/nominatim-db # Nominatim is now ready to use. The nominatim binary is available at # `$USERHOME/venv/bin/nominatim`. If you want to have 'nominatim' in your