From 4760e8341b85df3d2b6f8e62b561cf343dbce352 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 5 Dec 2024 15:19:36 +0100 Subject: [PATCH 01/17] move lua scripts into a separate directory --- .github/workflows/ci-tests.yml | 5 +---- {settings => lib-lua}/flex-base.lua | 0 {settings => lib-lua}/import-address.lua | 0 {settings => lib-lua}/import-admin.lua | 0 {settings => lib-lua}/import-extratags.lua | 0 {settings => lib-lua}/import-full.lua | 0 {settings => lib-lua}/import-street.lua | 0 {settings => lib-lua}/taginfo.lua | 3 ++- packaging/nominatim-api/extra_src/paths.py | 1 + packaging/nominatim-db/extra_src/nominatim_db/paths.py | 1 + packaging/nominatim-db/lib-lua | 1 + packaging/nominatim-db/pyproject.toml | 2 ++ src/nominatim_db/clicmd/args.py | 2 +- src/nominatim_db/config.py | 3 ++- src/nominatim_db/paths.py | 1 + test/bdd/steps/steps_osm_data.py | 2 +- test/python/config/test_config.py | 2 +- 17 files changed, 14 insertions(+), 9 deletions(-) rename {settings => lib-lua}/flex-base.lua (100%) rename {settings => lib-lua}/import-address.lua (100%) rename {settings => lib-lua}/import-admin.lua (100%) rename {settings => lib-lua}/import-extratags.lua (100%) rename {settings => lib-lua}/import-full.lua (100%) rename {settings => lib-lua}/import-street.lua (100%) rename {settings => lib-lua}/taginfo.lua (95%) create mode 120000 packaging/nominatim-db/lib-lua diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 8b38d3dc7..f453aefb5 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -185,9 +185,6 @@ jobs: - name: Prepare import environment run: | mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf - mv Nominatim/settings/flex-base.lua flex-base.lua - mv Nominatim/settings/import-extratags.lua import-extratags.lua - mv Nominatim/settings/taginfo.lua taginfo.lua rm -rf Nominatim mkdir data-env-reverse working-directory: /home/nominatim @@ -205,7 +202,7 @@ jobs: working-directory: /home/nominatim/nominatim-project - name: Print taginfo - run: lua taginfo.lua + run: lua ./nominatim-venv/lib/*/site-packages/nominatim_db/resources/lib-lua/taginfo.lua working-directory: /home/nominatim - name: Collect host OS information diff --git a/settings/flex-base.lua b/lib-lua/flex-base.lua similarity index 100% rename from settings/flex-base.lua rename to lib-lua/flex-base.lua diff --git a/settings/import-address.lua b/lib-lua/import-address.lua similarity index 100% rename from settings/import-address.lua rename to lib-lua/import-address.lua diff --git a/settings/import-admin.lua b/lib-lua/import-admin.lua similarity index 100% rename from settings/import-admin.lua rename to lib-lua/import-admin.lua diff --git a/settings/import-extratags.lua b/lib-lua/import-extratags.lua similarity index 100% rename from settings/import-extratags.lua rename to lib-lua/import-extratags.lua diff --git a/settings/import-full.lua b/lib-lua/import-full.lua similarity index 100% rename from settings/import-full.lua rename to lib-lua/import-full.lua diff --git a/settings/import-street.lua b/lib-lua/import-street.lua similarity index 100% rename from settings/import-street.lua rename to lib-lua/import-street.lua diff --git a/settings/taginfo.lua b/lib-lua/taginfo.lua similarity index 95% rename from settings/taginfo.lua rename to lib-lua/taginfo.lua index ef2ad2a60..fddaf298f 100644 --- a/settings/taginfo.lua +++ b/lib-lua/taginfo.lua @@ -6,7 +6,8 @@ osm2pgsql = {} function osm2pgsql.define_table(...) end -- provide path to flex-style lua file -flex = require('import-extratags') +package.path = arg[0]:match("(.*/)") .. "?.lua;" .. package.path +local flex = require('import-extratags') local json = require ('dkjson') diff --git a/packaging/nominatim-api/extra_src/paths.py b/packaging/nominatim-api/extra_src/paths.py index 797acbb55..7d186da16 100644 --- a/packaging/nominatim-api/extra_src/paths.py +++ b/packaging/nominatim-api/extra_src/paths.py @@ -11,4 +11,5 @@ DATA_DIR = None SQLLIB_DIR = None +LUALIB_DIR = None CONFIG_DIR = (Path(__file__) / '..' / 'resources' / 'settings').resolve() diff --git a/packaging/nominatim-db/extra_src/nominatim_db/paths.py b/packaging/nominatim-db/extra_src/nominatim_db/paths.py index 796ff08b3..02df50477 100644 --- a/packaging/nominatim-db/extra_src/nominatim_db/paths.py +++ b/packaging/nominatim-db/extra_src/nominatim_db/paths.py @@ -11,4 +11,5 @@ DATA_DIR = (Path(__file__) / '..' / 'resources').resolve() SQLLIB_DIR = (DATA_DIR / 'lib-sql') +LUALIB_DIR = (DATA_DIR / 'lib-lua') CONFIG_DIR = (DATA_DIR / 'settings') diff --git a/packaging/nominatim-db/lib-lua b/packaging/nominatim-db/lib-lua new file mode 120000 index 000000000..e4e1bd04c --- /dev/null +++ b/packaging/nominatim-db/lib-lua @@ -0,0 +1 @@ +../../lib-lua \ No newline at end of file diff --git a/packaging/nominatim-db/pyproject.toml b/packaging/nominatim-db/pyproject.toml index 841845f03..c34ce937e 100644 --- a/packaging/nominatim-db/pyproject.toml +++ b/packaging/nominatim-db/pyproject.toml @@ -44,6 +44,7 @@ include = [ "src/nominatim_db", "scripts", "lib-sql/**/*.sql", + "lib-lua/**/*.lua", "settings", "data/words.sql", "extra_src/nominatim_db/paths.py" @@ -65,6 +66,7 @@ packages = ["src/nominatim_db"] [tool.hatch.build.targets.wheel.force-include] "lib-sql" = "nominatim_db/resources/lib-sql" +"lib-lua" = "nominatim_db/resources/lib-lua" "settings" = "nominatim_db/resources/settings" "data/country_osm_grid.sql.gz" = "nominatim_db/resources/country_osm_grid.sql.gz" "data/words.sql" = "nominatim_db/resources/words.sql" diff --git a/src/nominatim_db/clicmd/args.py b/src/nominatim_db/clicmd/args.py index 488ecd184..a8ff210a2 100644 --- a/src/nominatim_db/clicmd/args.py +++ b/src/nominatim_db/clicmd/args.py @@ -189,7 +189,7 @@ def osm2pgsql_options(self, default_cache: int, return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.config.lib_dir.osm2pgsql, osm2pgsql_cache=self.osm2pgsql_cache or default_cache, osm2pgsql_style=self.config.get_import_style_file(), - osm2pgsql_style_path=self.config.config_dir, + osm2pgsql_style_path=self.config.lib_dir.lua, threads=self.threads or default_threads, dsn=self.config.get_libpq_dsn(), flatnode_file=str(self.config.get_path('FLATNODE_FILE') or ''), diff --git a/src/nominatim_db/config.py b/src/nominatim_db/config.py index ad54ab3d3..ae59cfd36 100644 --- a/src/nominatim_db/config.py +++ b/src/nominatim_db/config.py @@ -75,6 +75,7 @@ def __init__(self, project_dir: Optional[Union[Path, str]], class _LibDirs: osm2pgsql: Path sql = paths.SQLLIB_DIR + lua = paths.LUALIB_DIR data = paths.DATA_DIR self.lib_dir = _LibDirs() @@ -207,7 +208,7 @@ def get_import_style_file(self) -> Path: style = getattr(self, 'IMPORT_STYLE') if style in ('admin', 'street', 'address', 'full', 'extratags'): - return self.config_dir / f'import-{style}.lua' + return self.lib_dir.lua / f'import-{style}.lua' return self.find_config_file('', 'IMPORT_STYLE') diff --git a/src/nominatim_db/paths.py b/src/nominatim_db/paths.py index 2614fa145..ab34e4c0c 100644 --- a/src/nominatim_db/paths.py +++ b/src/nominatim_db/paths.py @@ -10,5 +10,6 @@ from pathlib import Path SQLLIB_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-sql').resolve() +LUALIB_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-lua').resolve() DATA_DIR = (Path(__file__) / '..' / '..' / '..' / 'data').resolve() CONFIG_DIR = (Path(__file__) / '..' / '..' / '..' / 'settings').resolve() diff --git a/test/bdd/steps/steps_osm_data.py b/test/bdd/steps/steps_osm_data.py index 4cee75f7a..70cf1515c 100644 --- a/test/bdd/steps/steps_osm_data.py +++ b/test/bdd/steps/steps_osm_data.py @@ -19,7 +19,7 @@ def get_osm2pgsql_options(nominatim_env, fname, append): osm2pgsql='osm2pgsql', osm2pgsql_cache=50, osm2pgsql_style=str(nominatim_env.get_test_config().get_import_style_file()), - osm2pgsql_style_path=nominatim_env.get_test_config().config_dir, + osm2pgsql_style_path=nominatim_env.get_test_config().lib_dir.lua, threads=1, dsn=nominatim_env.get_libpq_dsn(), flatnode_file='', diff --git a/test/python/config/test_config.py b/test/python/config/test_config.py index 8f90b5da1..9f68fcb9a 100644 --- a/test/python/config/test_config.py +++ b/test/python/config/test_config.py @@ -222,7 +222,7 @@ def test_get_import_style_intern(make_config, src_dir, monkeypatch): monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', 'street') - expected = src_dir / 'settings' / 'import-street.lua' + expected = src_dir / 'lib-lua' / 'import-street.lua' assert config.get_import_style_file() == expected From 438b8fed35ea5d250b826e9d6bd9d21f51ffb9d2 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 5 Dec 2024 16:24:44 +0100 Subject: [PATCH 02/17] convert flex-base.lua into a themepark theme This already allows to run Nominatim under themepark, currently as a topic-less theme. --- lib-lua/flex-base.lua | 556 +-------------------------- lib-lua/themes/nominatim/init.lua | 602 ++++++++++++++++++++++++++++++ 2 files changed, 608 insertions(+), 550 deletions(-) create mode 100644 lib-lua/themes/nominatim/init.lua diff --git a/lib-lua/flex-base.lua b/lib-lua/flex-base.lua index 4d960d726..9f9fda864 100644 --- a/lib-lua/flex-base.lua +++ b/lib-lua/flex-base.lua @@ -1,554 +1,10 @@ --- Core functions for Nominatim import flex style. --- +-- This is just an alias for the Nominatim themepark theme module +local flex = require('themes/nominatim/init') -local module = {} +function flex.load_topic(name, cfg) + local topic_file = debug.getinfo(1, "S").source:sub(2):match("(.*/)") .. 'themes/nominatim/topics/'.. name .. '.lua' -local PRE_DELETE = nil -local PRE_EXTRAS = nil -local POST_DELETE = nil -local MAIN_KEYS = nil -local NAMES = nil -local ADDRESS_TAGS = nil -local SAVE_EXTRA_MAINS = false -local POSTCODE_FALLBACK = true - --- tables required for taginfo -module.TAGINFO_MAIN = {keys = {}, delete_tags = {}} -module.TAGINFO_NAME_KEYS = {} -module.TAGINFO_ADDRESS_KEYS = {} - - --- The single place table. -local place_table = osm2pgsql.define_table{ - name = "place", - ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, - columns = { - { column = 'class', type = 'text', not_null = true }, - { column = 'type', type = 'text', not_null = true }, - { column = 'admin_level', type = 'smallint' }, - { column = 'name', type = 'hstore' }, - { column = 'address', type = 'hstore' }, - { column = 'extratags', type = 'hstore' }, - { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true }, - }, - data_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_DATA"), - index_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_INDEX"), - indexes = {} -} - ------------- Geometry functions for relations --------------------- - -function module.relation_as_multipolygon(o) - return o:as_multipolygon() -end - -function module.relation_as_multiline(o) - return o:as_multilinestring():line_merge() -end - - -module.RELATION_TYPES = { - multipolygon = module.relation_as_multipolygon, - boundary = module.relation_as_multipolygon, - waterway = module.relation_as_multiline -} - -------------- Place class ------------------------------------------ - -local Place = {} -Place.__index = Place - -function Place.new(object, geom_func) - local self = setmetatable({}, Place) - self.object = object - self.geom_func = geom_func - - self.admin_level = tonumber(self.object:grab_tag('admin_level')) - if self.admin_level == nil - or self.admin_level <= 0 or self.admin_level > 15 - or math.floor(self.admin_level) ~= self.admin_level then - self.admin_level = 15 - end - - self.num_entries = 0 - self.has_name = false - self.names = {} - self.address = {} - self.extratags = {} - - return self -end - -function Place:clean(data) - for k, v in pairs(self.object.tags) do - if data.delete ~= nil and data.delete(k, v) then - self.object.tags[k] = nil - elseif data.extra ~= nil and data.extra(k, v) then - self.extratags[k] = v - self.object.tags[k] = nil - end - end -end - -function Place:delete(data) - if data.match ~= nil then - for k, v in pairs(self.object.tags) do - if data.match(k, v) then - self.object.tags[k] = nil - end - end - end -end - -function Place:grab_extratags(data) - local count = 0 - - if data.match ~= nil then - for k, v in pairs(self.object.tags) do - if data.match(k, v) then - self.object.tags[k] = nil - self.extratags[k] = v - count = count + 1 - end - end - end - - return count -end - -local function strip_address_prefix(k) - if k:sub(1, 5) == 'addr:' then - return k:sub(6) - end - - if k:sub(1, 6) == 'is_in:' then - return k:sub(7) - end - - return k -end - - -function Place:grab_address_parts(data) - local count = 0 - - if data.groups ~= nil then - for k, v in pairs(self.object.tags) do - local atype = data.groups(k, v) - - if atype ~= nil then - if atype == 'main' then - self.has_name = true - self.address[strip_address_prefix(k)] = v - count = count + 1 - elseif atype == 'extra' then - self.address[strip_address_prefix(k)] = v - else - self.address[atype] = v - end - self.object.tags[k] = nil - end - end - end - - return count -end - - -function Place:grab_name_parts(data) - local fallback = nil - - if data.groups ~= nil then - for k, v in pairs(self.object.tags) do - local atype = data.groups(k, v) - - if atype ~= nil then - self.names[k] = v - self.object.tags[k] = nil - if atype == 'main' then - self.has_name = true - elseif atype == 'house' then - self.has_name = true - fallback = {'place', 'house', 'always'} - end - end - end - end - - return fallback -end - - -function Place:write_place(k, v, mtype, save_extra_mains) - if mtype == nil then - return 0 - end - - v = v or self.object.tags[k] - if v == nil then - return 0 - end - - if type(mtype) == 'table' then - mtype = mtype[v] or mtype[1] - end - - if mtype == 'always' or (self.has_name and mtype == 'named') then - return self:write_row(k, v, save_extra_mains) - end - - if mtype == 'named_with_key' then - local names = {} - local prefix = k .. ':name' - for namek, namev in pairs(self.object.tags) do - if namek:sub(1, #prefix) == prefix - and (#namek == #prefix - or namek:sub(#prefix + 1, #prefix + 1) == ':') then - names[namek:sub(#k + 2)] = namev - end - end - - if next(names) ~= nil then - local saved_names = self.names - self.names = names - - local results = self:write_row(k, v, save_extra_mains) - - self.names = saved_names - - return results - end - end - - return 0 -end - -function Place:write_row(k, v, save_extra_mains) - if self.geometry == nil then - self.geometry = self.geom_func(self.object) - end - if self.geometry:is_null() then - return 0 - end - - if save_extra_mains ~= nil then - for extra_k, extra_v in pairs(self.object.tags) do - if extra_k ~= k and save_extra_mains(extra_k, extra_v) then - self.extratags[extra_k] = extra_v - end - end - end - - place_table:insert{ - class = k, - type = v, - admin_level = self.admin_level, - name = next(self.names) and self.names, - address = next(self.address) and self.address, - extratags = next(self.extratags) and self.extratags, - geometry = self.geometry - } - - if save_extra_mains then - for tk, tv in pairs(self.object.tags) do - if save_extra_mains(tk, tv) then - self.extratags[tk] = nil - end - end - end - - self.num_entries = self.num_entries + 1 - - return 1 -end - - -function module.tag_match(data) - if data == nil or next(data) == nil then - return nil - end - - local fullmatches = {} - local key_prefixes = {} - local key_suffixes = {} - - if data.keys ~= nil then - for _, key in pairs(data.keys) do - if key:sub(1, 1) == '*' then - if #key > 1 then - if key_suffixes[#key - 1] == nil then - key_suffixes[#key - 1] = {} - end - key_suffixes[#key - 1][key:sub(2)] = true - end - elseif key:sub(#key, #key) == '*' then - if key_prefixes[#key - 1] == nil then - key_prefixes[#key - 1] = {} - end - key_prefixes[#key - 1][key:sub(1, #key - 1)] = true - else - fullmatches[key] = true - end - end - end - - if data.tags ~= nil then - for k, vlist in pairs(data.tags) do - if fullmatches[k] == nil then - fullmatches[k] = {} - for _, v in pairs(vlist) do - fullmatches[k][v] = true - end - end - end - end - - return function (k, v) - if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then - return true - end - - for slen, slist in pairs(key_suffixes) do - if #k >= slen and slist[k:sub(-slen)] ~= nil then - return true - end - end - - for slen, slist in pairs(key_prefixes) do - if #k >= slen and slist[k:sub(1, slen)] ~= nil then - return true - end - end - - return false - end -end - - -function module.tag_group(data) - if data == nil or next(data) == nil then - return nil - end - - local fullmatches = {} - local key_prefixes = {} - local key_suffixes = {} - - for group, tags in pairs(data) do - for _, key in pairs(tags) do - if key:sub(1, 1) == '*' then - if #key > 1 then - if key_suffixes[#key - 1] == nil then - key_suffixes[#key - 1] = {} - end - key_suffixes[#key - 1][key:sub(2)] = group - end - elseif key:sub(#key, #key) == '*' then - if key_prefixes[#key - 1] == nil then - key_prefixes[#key - 1] = {} - end - key_prefixes[#key - 1][key:sub(1, #key - 1)] = group - else - fullmatches[key] = group - end - end - end - - return function (k, v) - local val = fullmatches[k] - if val ~= nil then - return val - end - - for slen, slist in pairs(key_suffixes) do - if #k >= slen then - val = slist[k:sub(-slen)] - if val ~= nil then - return val - end - end - end - - for slen, slist in pairs(key_prefixes) do - if #k >= slen then - val = slist[k:sub(1, slen)] - if val ~= nil then - return val - end - end - end - end -end - --- Returns prefix part of the keys, and reject suffix matching keys -local function process_key(key) - if key:sub(1, 1) == '*' then - return nil - end - if key:sub(#key, #key) == '*' then - return key:sub(1, #key - 2) - end - return key -end - --- Process functions for all data types -function module.process_node(object) - - local function geom_func(o) - return o:as_point() - end - - module.process_tags(Place.new(object, geom_func)) -end - -function module.process_way(object) - - local function geom_func(o) - local geom = o:as_polygon() - - if geom:is_null() then - geom = o:as_linestring() - end - - return geom - end - - module.process_tags(Place.new(object, geom_func)) -end - -function module.process_relation(object) - local geom_func = module.RELATION_TYPES[object.tags.type] - - if geom_func ~= nil then - module.process_tags(Place.new(object, geom_func)) - end -end - --- The process functions are used by default by osm2pgsql. -osm2pgsql.process_node = module.process_node -osm2pgsql.process_way = module.process_way -osm2pgsql.process_relation = module.process_relation - -function module.process_tags(o) - o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS} - - -- Exception for boundary/place double tagging - if o.object.tags.boundary == 'administrative' then - o:grab_extratags{match = function (k, v) - return k == 'place' and v:sub(1,3) ~= 'isl' - end} - end - - -- name keys - local fallback = o:grab_name_parts{groups=NAMES} - - -- address keys - if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then - fallback = {'place', 'house', 'always'} - end - if o.address.country ~= nil and #o.address.country ~= 2 then - o.address['country'] = nil - end - if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then - fallback = {'place', 'postcode', 'always'} - end - - if o.address.interpolation ~= nil then - o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS) - return - end - - o:clean{delete = POST_DELETE} - - -- collect main keys - for k, v in pairs(o.object.tags) do - local ktype = MAIN_KEYS[k] - if ktype == 'fallback' then - if o.has_name then - fallback = {k, v, 'named'} - end - elseif ktype ~= nil then - o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS) - end - end - - if fallback ~= nil and o.num_entries == 0 then - o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS) - end -end - ---------- Convenience functions for simple style configuration ----------------- - - -function module.set_prefilters(data) - PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} - PRE_EXTRAS = module.tag_match{keys = data.extra_keys, - tags = data.extra_tags} - module.TAGINFO_MAIN.delete_tags = data.delete_tags -end - -function module.set_main_tags(data) - MAIN_KEYS = data - local keys = {} - for k, _ in pairs(data) do - table.insert(keys, k) - end - module.TAGINFO_MAIN.keys = keys -end - -function module.set_name_tags(data) - NAMES = module.tag_group(data) - - for _, lst in pairs(data) do - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_NAME_KEYS[key] = true - end - end - end -end - -function module.set_address_tags(data) - if data.postcode_fallback ~= nil then - POSTCODE_FALLBACK = data.postcode_fallback - data.postcode_fallback = nil - end - ADDRESS_TAGS = module.tag_group(data) - - for _, lst in pairs(data) do - if lst ~= nil then - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_ADDRESS_KEYS[key] = true - end - end - end - end -end - -function module.set_unused_handling(data) - if data.extra_keys == nil and data.extra_tags == nil then - POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} - SAVE_EXTRA_MAINS = function() return true end - elseif data.delete_keys == nil and data.delete_tags == nil then - POST_DELETE = nil - SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} - else - error("unused handler can have only 'extra_keys' or 'delete_keys' set.") - end -end - -function module.set_relation_types(data) - module.RELATION_TYPES = {} - for k, v in data do - if v == 'multipolygon' then - module.RELATION_TYPES[k] = module.relation_as_multipolygon - elseif v == 'multiline' then - module.RELATION_TYPES[k] = module.relation_as_multiline - end - end + loadfile(topic_file)(nil, flex, cfg or {}) end -return module +return flex diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua new file mode 100644 index 000000000..9b7882b3f --- /dev/null +++ b/lib-lua/themes/nominatim/init.lua @@ -0,0 +1,602 @@ +-- Nominatim themepark theme. +-- +-- The Nominatim theme creates a fixed set of import tables for use with +-- Nominatim. Creation and object processing are directly controlled by +-- the theme. Topics provide preset configurations. You should add exactly +-- one topic to your project. +-- +-- The theme also exports a number of functions that can be used to configure +-- its behaviour. These may be directly called in the style file after +-- importing the theme: +-- +-- local nominatim = themepark:init_theme('nominatim') +-- nominatim.set_main_tags{boundary = 'always'} +-- +-- This allows to write your own configuration from scratch. You can also +-- use it to customize topics. In that case, first add the topic, then +-- change the configuration: +-- +-- themepark:add_topic('nominatim/full') +-- local nominatim = themepark:init_theme('nominatim') +-- nominatim.ignore_tags{'amenity'} + +local module = {} + +local PRE_DELETE = nil +local PRE_EXTRAS = nil +local POST_DELETE = nil +local MAIN_KEYS = nil +local NAMES = nil +local ADDRESS_TAGS = nil +local SAVE_EXTRA_MAINS = false +local POSTCODE_FALLBACK = true + +-- This file can also be directly require'd instead of running it under +-- the themepark framework. In that case the first parameter is usually +-- the module name. Lets check for that, so that further down we can call +-- the low-level osm2pgsql functions instead of themepark functions. +local themepark = ... +if type(themepark) ~= 'table' then + themepark = nil +end + +-- tables required for taginfo +module.TAGINFO_MAIN = {keys = {}, delete_tags = {}} +module.TAGINFO_NAME_KEYS = {} +module.TAGINFO_ADDRESS_KEYS = {} + + +-- The single place table. +local place_table_definition = { + name = "place", + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'class', type = 'text', not_null = true }, + { column = 'type', type = 'text', not_null = true }, + { column = 'admin_level', type = 'smallint' }, + { column = 'name', type = 'hstore' }, + { column = 'address', type = 'hstore' }, + { column = 'extratags', type = 'hstore' }, + { column = 'geometry', type = 'geometry', projection = 'WGS84', not_null = true }, + }, + data_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_DATA"), + index_tablespace = os.getenv("NOMINATIM_TABLESPACE_PLACE_INDEX"), + indexes = {} +} + +local insert_row + +if themepark then + themepark:add_table(place_table_definition) + insert_row = function(columns) + themepark:insert('place', columns, {}, {}) + end +else + local place_table = osm2pgsql.define_table(place_table_definition) + insert_row = function(columns) + place_table:insert(columns) + end +end + +------------ Geometry functions for relations --------------------- + +function module.relation_as_multipolygon(o) + return o:as_multipolygon() +end + +function module.relation_as_multiline(o) + return o:as_multilinestring():line_merge() +end + + +module.RELATION_TYPES = { + multipolygon = module.relation_as_multipolygon, + boundary = module.relation_as_multipolygon, + waterway = module.relation_as_multiline +} + +------------- Place class ------------------------------------------ + +local Place = {} +Place.__index = Place + +function Place.new(object, geom_func) + local self = setmetatable({}, Place) + self.object = object + self.geom_func = geom_func + + self.admin_level = tonumber(self.object:grab_tag('admin_level')) + if self.admin_level == nil + or self.admin_level <= 0 or self.admin_level > 15 + or math.floor(self.admin_level) ~= self.admin_level then + self.admin_level = 15 + end + + self.num_entries = 0 + self.has_name = false + self.names = {} + self.address = {} + self.extratags = {} + + return self +end + +function Place:clean(data) + for k, v in pairs(self.object.tags) do + if data.delete ~= nil and data.delete(k, v) then + self.object.tags[k] = nil + elseif data.extra ~= nil and data.extra(k, v) then + self.extratags[k] = v + self.object.tags[k] = nil + end + end +end + +function Place:delete(data) + if data.match ~= nil then + for k, v in pairs(self.object.tags) do + if data.match(k, v) then + self.object.tags[k] = nil + end + end + end +end + +function Place:grab_extratags(data) + local count = 0 + + if data.match ~= nil then + for k, v in pairs(self.object.tags) do + if data.match(k, v) then + self.object.tags[k] = nil + self.extratags[k] = v + count = count + 1 + end + end + end + + return count +end + +local function strip_address_prefix(k) + if k:sub(1, 5) == 'addr:' then + return k:sub(6) + end + + if k:sub(1, 6) == 'is_in:' then + return k:sub(7) + end + + return k +end + + +function Place:grab_address_parts(data) + local count = 0 + + if data.groups ~= nil then + for k, v in pairs(self.object.tags) do + local atype = data.groups(k, v) + + if atype ~= nil then + if atype == 'main' then + self.has_name = true + self.address[strip_address_prefix(k)] = v + count = count + 1 + elseif atype == 'extra' then + self.address[strip_address_prefix(k)] = v + else + self.address[atype] = v + end + self.object.tags[k] = nil + end + end + end + + return count +end + + +function Place:grab_name_parts(data) + local fallback = nil + + if data.groups ~= nil then + for k, v in pairs(self.object.tags) do + local atype = data.groups(k, v) + + if atype ~= nil then + self.names[k] = v + self.object.tags[k] = nil + if atype == 'main' then + self.has_name = true + elseif atype == 'house' then + self.has_name = true + fallback = {'place', 'house', 'always'} + end + end + end + end + + return fallback +end + + +function Place:write_place(k, v, mtype, save_extra_mains) + if mtype == nil then + return 0 + end + + v = v or self.object.tags[k] + if v == nil then + return 0 + end + + if type(mtype) == 'table' then + mtype = mtype[v] or mtype[1] + end + + if mtype == 'always' or (self.has_name and mtype == 'named') then + return self:write_row(k, v, save_extra_mains) + end + + if mtype == 'named_with_key' then + local names = {} + local prefix = k .. ':name' + for namek, namev in pairs(self.object.tags) do + if namek:sub(1, #prefix) == prefix + and (#namek == #prefix + or namek:sub(#prefix + 1, #prefix + 1) == ':') then + names[namek:sub(#k + 2)] = namev + end + end + + if next(names) ~= nil then + local saved_names = self.names + self.names = names + + local results = self:write_row(k, v, save_extra_mains) + + self.names = saved_names + + return results + end + end + + return 0 +end + +function Place:write_row(k, v, save_extra_mains) + if self.geometry == nil then + self.geometry = self.geom_func(self.object) + end + if self.geometry:is_null() then + return 0 + end + + if save_extra_mains ~= nil then + for extra_k, extra_v in pairs(self.object.tags) do + if extra_k ~= k and save_extra_mains(extra_k, extra_v) then + self.extratags[extra_k] = extra_v + end + end + end + + insert_row{ + class = k, + type = v, + admin_level = self.admin_level, + name = next(self.names) and self.names, + address = next(self.address) and self.address, + extratags = next(self.extratags) and self.extratags, + geometry = self.geometry + } + + if save_extra_mains then + for tk, tv in pairs(self.object.tags) do + if save_extra_mains(tk, tv) then + self.extratags[tk] = nil + end + end + end + + self.num_entries = self.num_entries + 1 + + return 1 +end + + +function module.tag_match(data) + if data == nil or next(data) == nil then + return nil + end + + local fullmatches = {} + local key_prefixes = {} + local key_suffixes = {} + + if data.keys ~= nil then + for _, key in pairs(data.keys) do + if key:sub(1, 1) == '*' then + if #key > 1 then + if key_suffixes[#key - 1] == nil then + key_suffixes[#key - 1] = {} + end + key_suffixes[#key - 1][key:sub(2)] = true + end + elseif key:sub(#key, #key) == '*' then + if key_prefixes[#key - 1] == nil then + key_prefixes[#key - 1] = {} + end + key_prefixes[#key - 1][key:sub(1, #key - 1)] = true + else + fullmatches[key] = true + end + end + end + + if data.tags ~= nil then + for k, vlist in pairs(data.tags) do + if fullmatches[k] == nil then + fullmatches[k] = {} + for _, v in pairs(vlist) do + fullmatches[k][v] = true + end + end + end + end + + return function (k, v) + if fullmatches[k] ~= nil and (fullmatches[k] == true or fullmatches[k][v] ~= nil) then + return true + end + + for slen, slist in pairs(key_suffixes) do + if #k >= slen and slist[k:sub(-slen)] ~= nil then + return true + end + end + + for slen, slist in pairs(key_prefixes) do + if #k >= slen and slist[k:sub(1, slen)] ~= nil then + return true + end + end + + return false + end +end + + +function module.tag_group(data) + if data == nil or next(data) == nil then + return nil + end + + local fullmatches = {} + local key_prefixes = {} + local key_suffixes = {} + + for group, tags in pairs(data) do + for _, key in pairs(tags) do + if key:sub(1, 1) == '*' then + if #key > 1 then + if key_suffixes[#key - 1] == nil then + key_suffixes[#key - 1] = {} + end + key_suffixes[#key - 1][key:sub(2)] = group + end + elseif key:sub(#key, #key) == '*' then + if key_prefixes[#key - 1] == nil then + key_prefixes[#key - 1] = {} + end + key_prefixes[#key - 1][key:sub(1, #key - 1)] = group + else + fullmatches[key] = group + end + end + end + + return function (k, v) + local val = fullmatches[k] + if val ~= nil then + return val + end + + for slen, slist in pairs(key_suffixes) do + if #k >= slen then + val = slist[k:sub(-slen)] + if val ~= nil then + return val + end + end + end + + for slen, slist in pairs(key_prefixes) do + if #k >= slen then + val = slist[k:sub(1, slen)] + if val ~= nil then + return val + end + end + end + end +end + +-- Returns prefix part of the keys, and reject suffix matching keys +local function process_key(key) + if key:sub(1, 1) == '*' then + return nil + end + if key:sub(#key, #key) == '*' then + return key:sub(1, #key - 2) + end + return key +end + +-- Process functions for all data types +function module.process_node(object) + + local function geom_func(o) + return o:as_point() + end + + module.process_tags(Place.new(object, geom_func)) +end + +function module.process_way(object) + + local function geom_func(o) + local geom = o:as_polygon() + + if geom:is_null() then + geom = o:as_linestring() + end + + return geom + end + + module.process_tags(Place.new(object, geom_func)) +end + +function module.process_relation(object) + local geom_func = module.RELATION_TYPES[object.tags.type] + + if geom_func ~= nil then + module.process_tags(Place.new(object, geom_func)) + end +end + +-- The process functions are used by default by osm2pgsql. +if themepark then + themepark:add_proc('node', module.process_node) + themepark:add_proc('way', module.process_way) + themepark:add_proc('relation', module.process_relation) +else + osm2pgsql.process_node = module.process_node + osm2pgsql.process_way = module.process_way + osm2pgsql.process_relation = module.process_relation +end + +function module.process_tags(o) + o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS} + + -- Exception for boundary/place double tagging + if o.object.tags.boundary == 'administrative' then + o:grab_extratags{match = function (k, v) + return k == 'place' and v:sub(1,3) ~= 'isl' + end} + end + + -- name keys + local fallback = o:grab_name_parts{groups=NAMES} + + -- address keys + if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then + fallback = {'place', 'house', 'always'} + end + if o.address.country ~= nil and #o.address.country ~= 2 then + o.address['country'] = nil + end + if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then + fallback = {'place', 'postcode', 'always'} + end + + if o.address.interpolation ~= nil then + o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS) + return + end + + o:clean{delete = POST_DELETE} + + -- collect main keys + for k, v in pairs(o.object.tags) do + local ktype = MAIN_KEYS[k] + if ktype == 'fallback' then + if o.has_name then + fallback = {k, v, 'named'} + end + elseif ktype ~= nil then + o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS) + end + end + + if fallback ~= nil and o.num_entries == 0 then + o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS) + end +end + +--------- Convenience functions for simple style configuration ----------------- + + +function module.set_prefilters(data) + PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} + PRE_EXTRAS = module.tag_match{keys = data.extra_keys, + tags = data.extra_tags} + module.TAGINFO_MAIN.delete_tags = data.delete_tags +end + +function module.set_main_tags(data) + MAIN_KEYS = data + local keys = {} + for k, _ in pairs(data) do + table.insert(keys, k) + end + module.TAGINFO_MAIN.keys = keys +end + +function module.set_name_tags(data) + NAMES = module.tag_group(data) + + for _, lst in pairs(data) do + for _, k in ipairs(lst) do + local key = process_key(k) + if key ~= nil then + module.TAGINFO_NAME_KEYS[key] = true + end + end + end +end + +function module.set_address_tags(data) + if data.postcode_fallback ~= nil then + POSTCODE_FALLBACK = data.postcode_fallback + data.postcode_fallback = nil + end + ADDRESS_TAGS = module.tag_group(data) + + for _, lst in pairs(data) do + if lst ~= nil then + for _, k in ipairs(lst) do + local key = process_key(k) + if key ~= nil then + module.TAGINFO_ADDRESS_KEYS[key] = true + end + end + end + end +end + +function module.set_unused_handling(data) + if data.extra_keys == nil and data.extra_tags == nil then + POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} + SAVE_EXTRA_MAINS = function() return true end + elseif data.delete_keys == nil and data.delete_tags == nil then + POST_DELETE = nil + SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} + else + error("unused handler can have only 'extra_keys' or 'delete_keys' set.") + end +end + +function module.set_relation_types(data) + module.RELATION_TYPES = {} + for k, v in data do + if v == 'multipolygon' then + module.RELATION_TYPES[k] = module.relation_as_multipolygon + elseif v == 'multiline' then + module.RELATION_TYPES[k] = module.relation_as_multiline + end + end +end + +return module From 1eed2fa39517b83306d830137d6d49de13be1c52 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 5 Dec 2024 16:41:05 +0100 Subject: [PATCH 03/17] do not touch original tags of osm2pgsql OSM object --- lib-lua/themes/nominatim/init.lua | 49 ++++++++++++++++++------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 9b7882b3f..002a9e154 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -105,7 +105,7 @@ function Place.new(object, geom_func) self.object = object self.geom_func = geom_func - self.admin_level = tonumber(self.object:grab_tag('admin_level')) + self.admin_level = tonumber(self.object.tags.admin_level or 15) or 15 if self.admin_level == nil or self.admin_level <= 0 or self.admin_level > 15 or math.floor(self.admin_level) ~= self.admin_level then @@ -118,25 +118,36 @@ function Place.new(object, geom_func) self.address = {} self.extratags = {} + self.intags = {} + for k, v in pairs(self.object.tags) do + if PRE_DELETE ~= nil and PRE_DELETE(k, v) then + -- ignore + elseif PRE_EXTRAS ~= nil and PRE_EXTRAS(k, v) then + self.extratags[k] = v + elseif k ~= 'admin_level' then + self.intags[k] = v + end + end + return self end function Place:clean(data) - for k, v in pairs(self.object.tags) do + for k, v in pairs(self.intags) do if data.delete ~= nil and data.delete(k, v) then - self.object.tags[k] = nil + self.intags[k] = nil elseif data.extra ~= nil and data.extra(k, v) then self.extratags[k] = v - self.object.tags[k] = nil + self.intags[k] = nil end end end function Place:delete(data) if data.match ~= nil then - for k, v in pairs(self.object.tags) do + for k, v in pairs(self.intags) do if data.match(k, v) then - self.object.tags[k] = nil + self.intags[k] = nil end end end @@ -146,9 +157,9 @@ function Place:grab_extratags(data) local count = 0 if data.match ~= nil then - for k, v in pairs(self.object.tags) do + for k, v in pairs(self.intags) do if data.match(k, v) then - self.object.tags[k] = nil + self.intags[k] = nil self.extratags[k] = v count = count + 1 end @@ -175,7 +186,7 @@ function Place:grab_address_parts(data) local count = 0 if data.groups ~= nil then - for k, v in pairs(self.object.tags) do + for k, v in pairs(self.intags) do local atype = data.groups(k, v) if atype ~= nil then @@ -188,7 +199,7 @@ function Place:grab_address_parts(data) else self.address[atype] = v end - self.object.tags[k] = nil + self.intags[k] = nil end end end @@ -201,12 +212,12 @@ function Place:grab_name_parts(data) local fallback = nil if data.groups ~= nil then - for k, v in pairs(self.object.tags) do + for k, v in pairs(self.intags) do local atype = data.groups(k, v) if atype ~= nil then self.names[k] = v - self.object.tags[k] = nil + self.intags[k] = nil if atype == 'main' then self.has_name = true elseif atype == 'house' then @@ -226,7 +237,7 @@ function Place:write_place(k, v, mtype, save_extra_mains) return 0 end - v = v or self.object.tags[k] + v = v or self.intags[k] if v == nil then return 0 end @@ -242,7 +253,7 @@ function Place:write_place(k, v, mtype, save_extra_mains) if mtype == 'named_with_key' then local names = {} local prefix = k .. ':name' - for namek, namev in pairs(self.object.tags) do + for namek, namev in pairs(self.intags) do if namek:sub(1, #prefix) == prefix and (#namek == #prefix or namek:sub(#prefix + 1, #prefix + 1) == ':') then @@ -274,7 +285,7 @@ function Place:write_row(k, v, save_extra_mains) end if save_extra_mains ~= nil then - for extra_k, extra_v in pairs(self.object.tags) do + for extra_k, extra_v in pairs(self.intags) do if extra_k ~= k and save_extra_mains(extra_k, extra_v) then self.extratags[extra_k] = extra_v end @@ -292,7 +303,7 @@ function Place:write_row(k, v, save_extra_mains) } if save_extra_mains then - for tk, tv in pairs(self.object.tags) do + for tk, tv in pairs(self.intags) do if save_extra_mains(tk, tv) then self.extratags[tk] = nil end @@ -478,10 +489,8 @@ else end function module.process_tags(o) - o:clean{delete = PRE_DELETE, extra = PRE_EXTRAS} - -- Exception for boundary/place double tagging - if o.object.tags.boundary == 'administrative' then + if o.intags.boundary == 'administrative' then o:grab_extratags{match = function (k, v) return k == 'place' and v:sub(1,3) ~= 'isl' end} @@ -509,7 +518,7 @@ function module.process_tags(o) o:clean{delete = POST_DELETE} -- collect main keys - for k, v in pairs(o.object.tags) do + for k, v in pairs(o.intags) do local ktype = MAIN_KEYS[k] if ktype == 'fallback' then if o.has_name then From 70e351c528a38233d526fc60005f338bda40193e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 6 Dec 2024 09:17:33 +0100 Subject: [PATCH 04/17] osm2pgsql style: merge main tag and pre-filter handling Defining a tag as deleteable/extratag and main tag is mutually exclusive and deleting certain key/value combinations to exclude them from being used as a main tag is confusing. By merging the handling, such excludes can now be made explicit in the main list. By using the same lookup table, it is now also possible to have a short-cut for uninteresting objects. --- lib-lua/themes/nominatim/init.lua | 314 ++++++++++++++++++++++++------ 1 file changed, 251 insertions(+), 63 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 002a9e154..55634227f 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -22,10 +22,9 @@ local module = {} -local PRE_DELETE = nil -local PRE_EXTRAS = nil local POST_DELETE = nil -local MAIN_KEYS = nil +local MAIN_KEYS = {admin_level = {'delete'}} +local PRE_FILTER = {prefix = {}, suffix = {}} local NAMES = nil local ADDRESS_TAGS = nil local SAVE_EXTRA_MAINS = false @@ -95,6 +94,140 @@ module.RELATION_TYPES = { waterway = module.relation_as_multiline } +--------- Built-in place transformation functions -------------------------- + +local PlaceTransform = {} + +-- Special transform meanings which are interpreted elsewhere +PlaceTransform.fallback = 'fallback' +PlaceTransform.delete = 'delete' +PlaceTransform.extra = 'extra' + +-- always: unconditionally use that place +function PlaceTransform.always(place) + return place +end + +-- never: unconditionally drop the place +function PlaceTransform.never() + return nil +end + +-- named: use the place if it has a fully-qualified name +function PlaceTransform.named(place) + if place.has_name then + return place + end +end + +-- named_with_key: use place if there is a name with the main key prefix +function PlaceTransform.named_with_key(place, k) + local names = {} + local prefix = k .. ':name' + for namek, namev in pairs(place.intags) do + if namek:sub(1, #prefix) == prefix + and (#namek == #prefix + or namek:sub(#prefix + 1, #prefix + 1) == ':') then + names[namek:sub(#k + 2)] = namev + end + end + + if next(names) ~= nil then + return place:clone{names=names} + end +end + +----------------- other helper functions ----------------------------- + +local function lookup_prefilter_classification(k, v) + -- full matches + local desc = MAIN_KEYS[k] + local fullmatch = desc and (desc[v] or desc[1]) + if fullmatch ~= nil then + return fullmatch + end + -- suffixes + for slen, slist in pairs(PRE_FILTER.suffix) do + if #k >= slen then + local group = slist[k:sub(-slen)] + if group ~= nil then + return group + end + end + end + -- prefixes + for slen, slist in pairs(PRE_FILTER.prefix) do + if #k >= slen then + local group = slist[k:sub(1, slen)] + if group ~= nil then + return group + end + end + end +end + + +local function merge_filters_into_main(group, keys, tags) + if keys ~= nil then + for _, key in pairs(keys) do + -- ignore suffix and prefix matches + if key:sub(1, 1) ~= '*' and key:sub(#key, #key) ~= '*' then + if MAIN_KEYS[key] == nil then + MAIN_KEYS[key] = {} + end + MAIN_KEYS[key][1] = group + end + end + end + + if tags ~= nil then + for key, values in pairs(tags) do + if MAIN_KEYS[key] == nil then + MAIN_KEYS[key] = {} + end + for _, v in pairs(values) do + MAIN_KEYS[key][v] = group + end + end + end +end + + +local function remove_group_from_main(group) + for key, values in pairs(MAIN_KEYS) do + for _, ttype in pairs(values) do + if ttype == group then + values[ttype] = nil + end + end + if next(values) == nil then + MAIN_KEYS[key] = nil + end + end +end + + +local function add_pre_filter(data) + for group, keys in pairs(data) do + for _, key in pairs(keys) do + local klen = #key - 1 + if key:sub(1, 1) == '*' then + if klen > 0 then + if PRE_FILTER.suffix[klen] == nil then + PRE_FILTER.suffix[klen] = {} + end + PRE_FILTER.suffix[klen][key:sub(2)] = group + end + elseif key:sub(#key, #key) == '*' then + if PRE_FILTER.prefix[klen] == nil then + PRE_FILTER.prefix[klen] = {} + end + PRE_FILTER.prefix[klen][key:sub(1, klen)] = group + end + end + end +end + ------------- Place class ------------------------------------------ local Place = {} @@ -119,16 +252,25 @@ function Place.new(object, geom_func) self.extratags = {} self.intags = {} + + local has_main_tags = false for k, v in pairs(self.object.tags) do - if PRE_DELETE ~= nil and PRE_DELETE(k, v) then - -- ignore - elseif PRE_EXTRAS ~= nil and PRE_EXTRAS(k, v) then + local group = lookup_prefilter_classification(k, v) + if group == 'extra' then self.extratags[k] = v - elseif k ~= 'admin_level' then + elseif group ~= 'delete' then self.intags[k] = v + if group ~= nil then + has_main_tags = true + end end end + if not has_main_tags then + -- no interesting tags, don't bother processing + self.intags = {} + end + return self end @@ -222,7 +364,7 @@ function Place:grab_name_parts(data) self.has_name = true elseif atype == 'house' then self.has_name = true - fallback = {'place', 'house', 'always'} + fallback = {'place', 'house', PlaceTransform.always} end end end @@ -232,45 +374,17 @@ function Place:grab_name_parts(data) end -function Place:write_place(k, v, mtype, save_extra_mains) - if mtype == nil then - return 0 - end - +function Place:write_place(k, v, mfunc, save_extra_mains) v = v or self.intags[k] if v == nil then return 0 end - if type(mtype) == 'table' then - mtype = mtype[v] or mtype[1] - end - - if mtype == 'always' or (self.has_name and mtype == 'named') then - return self:write_row(k, v, save_extra_mains) - end - - if mtype == 'named_with_key' then - local names = {} - local prefix = k .. ':name' - for namek, namev in pairs(self.intags) do - if namek:sub(1, #prefix) == prefix - and (#namek == #prefix - or namek:sub(#prefix + 1, #prefix + 1) == ':') then - names[namek:sub(#k + 2)] = namev - end - end - - if next(names) ~= nil then - local saved_names = self.names - self.names = names - - local results = self:write_row(k, v, save_extra_mains) - - self.names = saved_names - - return results - end + local place = mfunc(self, k, v) + if place then + local res = place:write_row(k, v, save_extra_mains) + self.num_entries = self.num_entries + res + return res end return 0 @@ -310,12 +424,25 @@ function Place:write_row(k, v, save_extra_mains) end end - self.num_entries = self.num_entries + 1 - return 1 end +function Place:clone(data) + local cp = setmetatable({}, Place) + cp.object = self.object + cp.geometry = data.geometry or self.geometry + cp.geom_func = self.geom_func + cp.intags = data.intags or self.intags + cp.admin_level = data.admin_level or self.admin_level + cp.names = data.names or self.names + cp.address = data.address or self.address + cp.extratags = data.extratags or self.extratags + + return cp +end + + function module.tag_match(data) if data == nil or next(data) == nil then return nil @@ -489,6 +616,10 @@ else end function module.process_tags(o) + if next(o.intags) == nil then + return -- shortcut when pre-filtering has removed all tags + end + -- Exception for boundary/place double tagging if o.intags.boundary == 'administrative' then o:grab_extratags{match = function (k, v) @@ -501,17 +632,17 @@ function module.process_tags(o) -- address keys if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then - fallback = {'place', 'house', 'always'} + fallback = {'place', 'house', PlaceTransform.always} end if o.address.country ~= nil and #o.address.country ~= 2 then o.address['country'] = nil end if POSTCODE_FALLBACK and fallback == nil and o.address.postcode ~= nil then - fallback = {'place', 'postcode', 'always'} + fallback = {'place', 'postcode', PlaceTransform.always} end if o.address.interpolation ~= nil then - o:write_place('place', 'houses', 'always', SAVE_EXTRA_MAINS) + o:write_place('place', 'houses', PlaceTransform.always, SAVE_EXTRA_MAINS) return end @@ -519,13 +650,14 @@ function module.process_tags(o) -- collect main keys for k, v in pairs(o.intags) do - local ktype = MAIN_KEYS[k] - if ktype == 'fallback' then - if o.has_name then - fallback = {k, v, 'named'} + local ktable = MAIN_KEYS[k] + if ktable then + local ktype = ktable[v] or ktable[1] + if type(ktype) == 'function' then + o:write_place(k, v, ktype, SAVE_EXTRA_MAINS) + elseif ktype == 'fallback' and o.has_name then + fallback = {k, v, PlaceTransform.named} end - elseif ktype ~= nil then - o:write_place(k, v, MAIN_KEYS[k], SAVE_EXTRA_MAINS) end end @@ -536,23 +668,67 @@ end --------- Convenience functions for simple style configuration ----------------- - function module.set_prefilters(data) - PRE_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} - PRE_EXTRAS = module.tag_match{keys = data.extra_keys, - tags = data.extra_tags} - module.TAGINFO_MAIN.delete_tags = data.delete_tags + remove_group_from_main('delete') + merge_filters_into_main('delete', data.delete_keys, data.delete_tags) + + remove_group_from_main('extra') + merge_filters_into_main('extra', data.extra_keys, data.extra_tags) + + PRE_FILTER = {prefix = {}, suffix = {}} + add_pre_filter{delete = data.delete_keys, extra = data.extra_keys} end + +function module.ignore_tags(data) + merge_filters_into_main('delete', data) + add_pre_filter{delete = data} +end + + +function module.add_for_extratags(data) + merge_filters_into_main('extra', data) + add_pre_filter{extra = data} +end + + function module.set_main_tags(data) - MAIN_KEYS = data - local keys = {} - for k, _ in pairs(data) do - table.insert(keys, k) + for key, values in pairs(MAIN_KEYS) do + for _, ttype in pairs(values) do + if ttype == 'fallback' or type(ttype) == 'function' then + values[ttype] = nil + end + end + if next(values) == nil then + MAIN_KEYS[key] = nil + end + end + module.add_main_tags(data) +end + + +function module.add_main_tags(data) + for k, v in pairs(data) do + if MAIN_KEYS[k] == nil then + MAIN_KEYS[k] = {} + end + if type(v) == 'function' then + MAIN_KEYS[k][1] = v + elseif type(v) == 'string' then + MAIN_KEYS[k][1] = PlaceTransform[v] + elseif type(v) == 'table' then + for subk, subv in pairs(v) do + if type(subv) == 'function' then + MAIN_KEYS[k][subk] = subv + else + MAIN_KEYS[k][subk] = PlaceTransform[subv] + end + end + end end - module.TAGINFO_MAIN.keys = keys end + function module.set_name_tags(data) NAMES = module.tag_group(data) @@ -564,8 +740,11 @@ function module.set_name_tags(data) end end end + remove_group_from_main('fallback:name') + merge_filters_into_main('fallback:name', data.house) end + function module.set_address_tags(data) if data.postcode_fallback ~= nil then POSTCODE_FALLBACK = data.postcode_fallback @@ -583,8 +762,17 @@ function module.set_address_tags(data) end end end + + remove_group_from_main('fallback:address') + remove_group_from_main('fallback:postcode') + merge_filters_into_main('fallback:address', data.main) + if POSTCODE_FALLBACK then + merge_filters_into_main('fallback:postcode', data.postcode) + end + merge_filters_into_main('fallback:address', data.interpolation) end + function module.set_unused_handling(data) if data.extra_keys == nil and data.extra_tags == nil then POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} From 59416178bd3962aab53e87f3582c58e1361db801 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 6 Dec 2024 09:31:50 +0100 Subject: [PATCH 05/17] osm2pgsql style: simplify computation of extra tags Now implemented as a simple filter function which can also be customized by the user. --- lib-lua/themes/nominatim/init.lua | 104 ++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 33 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 55634227f..c2dfb130c 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -22,12 +22,13 @@ local module = {} -local POST_DELETE = nil local MAIN_KEYS = {admin_level = {'delete'}} local PRE_FILTER = {prefix = {}, suffix = {}} -local NAMES = nil -local ADDRESS_TAGS = nil -local SAVE_EXTRA_MAINS = false +local NAMES = {} +local NAME_FILTER = nil +local ADDRESS_TAGS = {} +local ADDRESS_FILTER = nil +local EXTRATAGS_FILTER local POSTCODE_FALLBACK = true -- This file can also be directly require'd instead of running it under @@ -137,6 +138,24 @@ function PlaceTransform.named_with_key(place, k) end end +--------- Built-in extratags transformation functions --------------- + +local function default_extratags_filter(p, k) + -- Default handling is to copy over place tag for boundaries. + -- Nominatim needs this. + if k ~= 'boundary' or p.intags.place == nil then + return p.extratags + end + + local extra = { place = p.intags.place } + for kin, vin in pairs(p.extratags) do + extra[kin] = vin + end + + return extra +end +EXTRATAGS_FILTER = default_extratags_filter + ----------------- other helper functions ----------------------------- local function lookup_prefilter_classification(k, v) @@ -374,7 +393,7 @@ function Place:grab_name_parts(data) end -function Place:write_place(k, v, mfunc, save_extra_mains) +function Place:write_place(k, v, mfunc) v = v or self.intags[k] if v == nil then return 0 @@ -382,7 +401,7 @@ function Place:write_place(k, v, mfunc, save_extra_mains) local place = mfunc(self, k, v) if place then - local res = place:write_row(k, v, save_extra_mains) + local res = place:write_row(k, v) self.num_entries = self.num_entries + res return res end @@ -390,7 +409,7 @@ function Place:write_place(k, v, mfunc, save_extra_mains) return 0 end -function Place:write_row(k, v, save_extra_mains) +function Place:write_row(k, v) if self.geometry == nil then self.geometry = self.geom_func(self.object) end @@ -398,12 +417,9 @@ function Place:write_row(k, v, save_extra_mains) return 0 end - if save_extra_mains ~= nil then - for extra_k, extra_v in pairs(self.intags) do - if extra_k ~= k and save_extra_mains(extra_k, extra_v) then - self.extratags[extra_k] = extra_v - end - end + local extratags = EXTRATAGS_FILTER(self, k, v) + if not (extratags and next(extratags)) then + extratags = nil end insert_row{ @@ -412,18 +428,10 @@ function Place:write_row(k, v, save_extra_mains) admin_level = self.admin_level, name = next(self.names) and self.names, address = next(self.address) and self.address, - extratags = next(self.extratags) and self.extratags, + extratags = extratags, geometry = self.geometry } - if save_extra_mains then - for tk, tv in pairs(self.intags) do - if save_extra_mains(tk, tv) then - self.extratags[tk] = nil - end - end - end - return 1 end @@ -534,7 +542,7 @@ function module.tag_group(data) end end - return function (k, v) + return function (k) local val = fullmatches[k] if val ~= nil then return val @@ -642,19 +650,17 @@ function module.process_tags(o) end if o.address.interpolation ~= nil then - o:write_place('place', 'houses', PlaceTransform.always, SAVE_EXTRA_MAINS) + o:write_place('place', 'houses', PlaceTransform.always) return end - o:clean{delete = POST_DELETE} - -- collect main keys for k, v in pairs(o.intags) do local ktable = MAIN_KEYS[k] if ktable then local ktype = ktable[v] or ktable[1] if type(ktype) == 'function' then - o:write_place(k, v, ktype, SAVE_EXTRA_MAINS) + o:write_place(k, v, ktype) elseif ktype == 'fallback' and o.has_name then fallback = {k, v, PlaceTransform.named} end @@ -662,7 +668,7 @@ function module.process_tags(o) end if fallback ~= nil and o.num_entries == 0 then - o:write_place(fallback[1], fallback[2], fallback[3], SAVE_EXTRA_MAINS) + o:write_place(fallback[1], fallback[2], fallback[3]) end end @@ -774,12 +780,44 @@ end function module.set_unused_handling(data) - if data.extra_keys == nil and data.extra_tags == nil then - POST_DELETE = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} - SAVE_EXTRA_MAINS = function() return true end + if type(data) == 'function' then + EXTRATAGS_FILTER = data + elseif data == nil then + EXTRATAGS_FILTER = default_extratags_filter + elseif data.extra_keys == nil and data.extra_tags == nil then + local delfilter = module.tag_match{keys = data.delete_keys, tags = data.delete_tags} + EXTRATAGS_FILTER = function (p, k) + local extra = {} + for kin, vin in pairs(p.intags) do + if kin ~= k and not delfilter(kin, vin) then + extra[kin] = vin + end + end + if next(extra) == nil then + return p.extratags + end + for kextra, vextra in pairs(p.extratags) do + extra[kextra] = vextra + end + return extra + end elseif data.delete_keys == nil and data.delete_tags == nil then - POST_DELETE = nil - SAVE_EXTRA_MAINS = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} + local incfilter = module.tag_match{keys = data.extra_keys, tags = data.extra_tags} + EXTRATAGS_FILTER = function (p, k) + local extra = {} + for kin, vin in pairs(p.intags) do + if kin ~= k and incfilter(kin, vin) then + extra[kin] = vin + end + end + if next(extra) == nil then + return p.extratags + end + for kextra, vextra in pairs(p.extratags) do + extra[kextra] = vextra + end + return extra + end else error("unused handler can have only 'extra_keys' or 'delete_keys' set.") end From d1b7c14f7987a20c218c96fe8071427536f8b36e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 6 Dec 2024 11:00:25 +0100 Subject: [PATCH 06/17] osm2pgsql style: add modification for name and address, with tests --- lib-lua/themes/nominatim/init.lua | 82 +++++++----- .../bdd/osm2pgsql/import/custom_style.feature | 123 +++++++++++++++++- 2 files changed, 174 insertions(+), 31 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index c2dfb130c..f8706bb92 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -636,10 +636,10 @@ function module.process_tags(o) end -- name keys - local fallback = o:grab_name_parts{groups=NAMES} + local fallback = o:grab_name_parts{groups=NAME_FILTER} -- address keys - if o:grab_address_parts{groups=ADDRESS_TAGS} > 0 and fallback == nil then + if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then fallback = {'place', 'house', PlaceTransform.always} end if o.address.country ~= nil and #o.address.country ~= 2 then @@ -686,7 +686,7 @@ function module.set_prefilters(data) end -function module.ignore_tags(data) +function module.ignore_keys(data) merge_filters_into_main('delete', data) add_pre_filter{delete = data} end @@ -735,47 +735,71 @@ function module.add_main_tags(data) end -function module.set_name_tags(data) - NAMES = module.tag_group(data) - - for _, lst in pairs(data) do - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_NAME_KEYS[key] = true - end +function module.modify_name_tags(data) + for k,v in pairs(data) do + if next(v) then + NAMES[k] = v + else + NAMES[k] = nil end end + NAME_FILTER = module.tag_group(NAMES) remove_group_from_main('fallback:name') - merge_filters_into_main('fallback:name', data.house) + if data.house ~= nil then + merge_filters_into_main('fallback:name', data.house) + end +end + + +function module.set_name_tags(data) + NAMES = {} + module.modify_name_tags(data) end function module.set_address_tags(data) - if data.postcode_fallback ~= nil then - POSTCODE_FALLBACK = data.postcode_fallback - data.postcode_fallback = nil - end - ADDRESS_TAGS = module.tag_group(data) - - for _, lst in pairs(data) do - if lst ~= nil then - for _, k in ipairs(lst) do - local key = process_key(k) - if key ~= nil then - module.TAGINFO_ADDRESS_KEYS[key] = true - end - end + ADDRESS_TAGS = {} + module.modify_address_tags(data) +end + + +function module.modify_address_tags(data) + for k, v in pairs(data) do + if k == 'postcode_fallback' then + POSTCODE_FALLBACK = v + elseif next(v) == nil then + ADDRESS_TAGS[k] = nil + else + ADDRESS_TAGS[k] = v end end + ADDRESS_FILTER = module.tag_group(ADDRESS_TAGS) + remove_group_from_main('fallback:address') - remove_group_from_main('fallback:postcode') merge_filters_into_main('fallback:address', data.main) + merge_filters_into_main('fallback:address', data.interpolation) + remove_group_from_main('fallback:postcode') if POSTCODE_FALLBACK then merge_filters_into_main('fallback:postcode', data.postcode) end - merge_filters_into_main('fallback:address', data.interpolation) +end + + +function module.set_address_tags(data) + ADDRESS_TAGS_SOURCE = {} + module.modify_address_tags(data) +end + + +function module.set_postcode_fallback(enable) + if POSTCODE_FALLBACK ~= enable then + remove_group_from_main('fallback:postcode') + if enable then + merge_filters_into_main('fallback:postcode', ADDRESS_TAGS.postcode) + end + end + POSTCODE_FALLBACK = enable end diff --git a/test/bdd/osm2pgsql/import/custom_style.feature b/test/bdd/osm2pgsql/import/custom_style.feature index 2ca95c91a..405afdb5c 100644 --- a/test/bdd/osm2pgsql/import/custom_style.feature +++ b/test/bdd/osm2pgsql/import/custom_style.feature @@ -2,7 +2,7 @@ Feature: Import with custom styles by osm2pgsql Tests for the example customizations given in the documentation. - Scenario: Custom main tags + Scenario: Custom main tags (set new ones) Given the lua style file """ local flex = require('import-full') @@ -28,6 +28,35 @@ Feature: Import with custom styles by osm2pgsql | N13 | highway | primary | | N15 | highway | primary | + Scenario: Custom main tags (modify existing) + Given the lua style file + """ + local flex = require('import-full') + + flex.add_main_tags{ + amenity = {prison = 'delete'}, + highway = {stop = 'named'}, + aeroway = 'named' + } + """ + When loading osm data + """ + n10 Tamenity=hotel x0 y0 + n11 Tamenity=prison x0 y0 + n12 Thighway=stop x0 y0 + n13 Thighway=stop,name=BigStop x0 y0 + n14 Thighway=give_way x0 y0 + n15 Thighway=bus_stop x0 y0 + n16 Taeroway=no,name=foo x0 y0 + n17 Taeroway=taxiway,name=D15 x0 y0 + """ + Then place contains exactly + | object | class | type | + | N10 | amenity | hotel | + | N13 | highway | stop | + | N15 | highway | bus_stop | + | N17 | aeroway | taxiway | + Scenario: Prefiltering tags Given the lua style file """ @@ -56,6 +85,38 @@ Feature: Import with custom styles by osm2pgsql | N4:tourism | - | | N4:amenity | - | + Scenario: Ignore some tags + Given the lua style file + """ + local flex = require('import-extratags') + + flex.ignore_keys{'ref:*', 'surface'} + """ + When loading osm data + """ + n100 Thighway=residential,ref=34,ref:bodo=34,surface=gray,extra=1 x0 y0 + """ + Then place contains exactly + | object | name | extratags | + | N100 | 'ref' : '34' | 'extra': '1' | + + + Scenario: Add for extratags + Given the lua style file + """ + local flex = require('import-full') + + flex.add_for_extratags{'ref:*', 'surface'} + """ + When loading osm data + """ + n100 Thighway=residential,ref=34,ref:bodo=34,surface=gray,extra=1 x0 y0 + """ + Then place contains exactly + | object | name | extratags | + | N100 | 'ref' : '34' | 'ref:bodo': '34', 'surface': 'gray' | + + Scenario: Name tags Given the lua style file """ @@ -78,6 +139,22 @@ Feature: Import with custom styles by osm2pgsql | N3:highway | 'name': 'Greens' | | N4:highway | 'name': 'Red', 'ref': '45' | + Scenario: Modify name tags + Given the lua style file + """ + local flex = require('import-full') + + flex.modify_name_tags{house = {}, extra = {'o'}} + """ + When loading osm data + """ + n1 Ttourism=hotel,ref=45,o=good + n2 Taddr:housename=Old,addr:street=Away + """ + Then place contains exactly + | object | name | + | N1:tourism | 'o': 'good' | + Scenario: Address tags Given the lua style file """ @@ -101,7 +178,24 @@ Feature: Import with custom styles by osm2pgsql | N1:tourism | hotel | 'street': 'Foo' | | N2:place | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' | - Scenario: Unused handling + Scenario: Modify address tags + Given the lua style file + """ + local flex = require('import-full') + + flex.set_address_tags{ + extra = {'addr:*'}, + } + """ + When loading osm data + """ + n2 Taddr:housenumber=23,addr:street=Budd,is_in:city=Faraway,postal_code=5567 x0 y0 + """ + Then place contains exactly + | object | type | address | + | N2:place | house | 'housenumber': '23', 'street': 'Budd', 'postcode': '5567' | + + Scenario: Unused handling (delete) Given the lua style file """ local flex = require('import-full') @@ -122,6 +216,31 @@ Feature: Import with custom styles by osm2pgsql | N1:tourism | hotel | 'tiger:county': 'Fargo' | - | | N2:tourism | hotel | - | 'else': 'other' | + Scenario: Unused handling (extra) + Given the lua style file + """ + local flex = require('flex-base') + flex.set_main_tags{highway = 'always', + wikipedia = 'extra'} + flex.add_for_extratags{'wikipedia:*', 'wikidata'} + flex.set_unused_handling{extra_keys = {'surface'}} + """ + When loading osm data + """ + n100 Thighway=path,foo=bar,wikipedia=en:Path x0 y0 + n234 Thighway=path,surface=rough x0 y0 + n445 Thighway=path,name=something x0 y0 + n446 Thighway=path,wikipedia:en=Path,wikidata=Q23 x0 y0 + n567 Thighway=path,surface=dirt,wikipedia:en=Path x0 y0 + """ + Then place contains exactly + | object | type | extratags | + | N100:highway | path | 'wikipedia': 'en:Path' | + | N234:highway | path | 'surface': 'rough' | + | N445:highway | path | - | + | N446:highway | path | 'wikipedia:en': 'Path', 'wikidata': 'Q23' | + | N567:highway | path | 'surface': 'dirt', 'wikipedia:en': 'Path' | + Scenario: Additional relation types Given the lua style file """ From 59bce26afef3fd65e606e09979c11733dfac6966 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 6 Dec 2024 12:12:27 +0100 Subject: [PATCH 07/17] convert import styles to themepark Introduces presets which avoid much of the previous configuration duplication. The original import files are now thin wrappers around the themepark themes. --- lib-lua/flex-base.lua | 4 + lib-lua/import-address.lua | 72 +---- lib-lua/import-admin.lua | 70 +---- lib-lua/import-extratags.lua | 124 +------- lib-lua/import-full.lua | 124 +------- lib-lua/import-street.lua | 72 +---- lib-lua/themes/nominatim/init.lua | 50 +++- lib-lua/themes/nominatim/presets.lua | 281 ++++++++++++++++++ lib-lua/themes/nominatim/topics/address.lua | 23 ++ lib-lua/themes/nominatim/topics/admin.lua | 20 ++ lib-lua/themes/nominatim/topics/full.lua | 32 ++ lib-lua/themes/nominatim/topics/street.lua | 22 ++ test/bdd/api/details/simple.feature | 1 - .../bdd/osm2pgsql/import/custom_style.feature | 2 +- test/bdd/osm2pgsql/import/tags.feature | 2 +- 15 files changed, 442 insertions(+), 457 deletions(-) create mode 100644 lib-lua/themes/nominatim/presets.lua create mode 100644 lib-lua/themes/nominatim/topics/address.lua create mode 100644 lib-lua/themes/nominatim/topics/admin.lua create mode 100644 lib-lua/themes/nominatim/topics/full.lua create mode 100644 lib-lua/themes/nominatim/topics/street.lua diff --git a/lib-lua/flex-base.lua b/lib-lua/flex-base.lua index 9f9fda864..1173c53ff 100644 --- a/lib-lua/flex-base.lua +++ b/lib-lua/flex-base.lua @@ -4,6 +4,10 @@ local flex = require('themes/nominatim/init') function flex.load_topic(name, cfg) local topic_file = debug.getinfo(1, "S").source:sub(2):match("(.*/)") .. 'themes/nominatim/topics/'.. name .. '.lua' + if topic_file == nil then + error('Cannot find topic: ' .. name) + end + loadfile(topic_file)(nil, flex, cfg or {}) end diff --git a/lib-lua/import-address.lua b/lib-lua/import-address.lua index b177b73ca..bec21505a 100644 --- a/lib-lua/import-address.lua +++ b/lib-lua/import-address.lua @@ -1,74 +1,6 @@ +-- This is just an alias for the Nominatim themepark address topic local flex = require('flex-base') -flex.set_main_tags{ - highway = {motorway = 'always', - trunk = 'always', - primary = 'always', - secondary = 'always', - tertiary = 'always', - unclassified = 'always', - residential = 'always', - road = 'always', - living_street = 'always', - pedestrian = 'always', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - boundary = {administrative = 'named', - postal_code = 'always'}, - landuse = 'fallback', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'building', 'source', - 'source', '*source', 'type', - 'is_in:postcode', '*:wikidata', '*:wikipedia', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {landuse = {'cemetry', 'no'}, - boundary = {'place'}}, - extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}, - house = {'addr:housename'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'} - } - - -flex.set_unused_handling{extra_keys = {'place'}} +flex.load_topic('address') return flex diff --git a/lib-lua/import-admin.lua b/lib-lua/import-admin.lua index 78eac5f52..8d1230a11 100644 --- a/lib-lua/import-admin.lua +++ b/lib-lua/import-admin.lua @@ -1,72 +1,6 @@ +-- This is just an alias for the Nominatim themepark admin topic local flex = require('flex-base') -flex.set_main_tags{ - boundary = {administrative = 'named'}, - landuse = {residential = 'fallback', - farm = 'fallback', - farmyard = 'fallback', - industrial = 'fallback', - commercial = 'fallback', - allotments = 'fallback', - retail = 'fallback'}, - place = {county = 'always', - district = 'always', - municipality = 'always', - city = 'always', - town = 'always', - borough = 'always', - village = 'always', - suburb = 'always', - hamlet = 'always', - croft = 'always', - subdivision = 'always', - allotments = 'always', - neighbourhood = 'always', - quarter = 'always', - isolated_dwelling = 'always', - farm = 'always', - city_block = 'always', - mountain_pass = 'always', - square = 'always', - locality = 'always'} -} - -flex.set_prefilters{delete_keys = {'building', 'source', 'highway', - 'addr:housenumber', 'addr:street', 'addr:city', - 'addr:interpolation', - 'source', '*source', 'type', - 'is_in:postcode', '*:wikidata', '*:wikipedia', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {landuse = {'cemetry', 'no'}, - boundary = {'place'}}, - extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'} - } - -flex.set_address_tags{extra = {'addr:*', 'is_in:*'}, - postcode = {'postal_code', 'postcode', 'addr:postcode'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - postcode_fallback = false - } - -flex.set_unused_handling{extra_keys = {'place'}} +flex.load_topic('admin') return flex diff --git a/lib-lua/import-extratags.lua b/lib-lua/import-extratags.lua index 0d242b0f3..53b1c81e6 100644 --- a/lib-lua/import-extratags.lua +++ b/lib-lua/import-extratags.lua @@ -1,126 +1,6 @@ +-- This is just an alias for the Nominatim themepark full topic local flex = require('flex-base') -flex.set_main_tags{ - building = 'fallback', - emergency = 'always', - healthcare = 'fallback', - historic = 'always', - military = 'always', - natural = 'named', - highway = {'always', - street_lamp = 'named', - traffic_signals = 'named', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - railway = 'named', - man_made = {'none', - pier = 'always', - tower = 'always', - bridge = 'always', - works = 'named', - water_tower = 'always', - dyke = 'named', - adit = 'named', - lighthouse = 'always', - watermill = 'always', - tunnel = 'always'}, - aerialway = 'always', - boundary = {'named', - postal_code = 'always'}, - aeroway = 'always', - amenity = 'always', - club = 'always', - craft = 'always', - junction = 'fallback', - landuse = 'fallback', - leisure = {'always', - nature_reserve = 'fallback'}, - office = 'always', - mountain_pass = 'always', - shop = 'always', - tourism = 'always', - bridge = 'named_with_key', - tunnel = 'named_with_key', - waterway = 'named', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution', - 'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*', - 'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*', - 'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*', - 'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type', - 'ref:linz:*', 'is_in:postcode'}, - delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'}, - historic = {'yes', 'no'}, - military = {'yes', 'no'}, - natural = {'yes', 'no', 'coastline'}, - highway = {'no', 'turning_circle', 'mini_roundabout', - 'noexit', 'crossing', 'give_way', 'stop'}, - railway = {'level_crossing', 'no', 'rail', 'switch', - 'abandoned', 'signal', 'buffer_stop', 'razed'}, - aerialway = {'pylon', 'no'}, - aeroway = {'no'}, - amenity = {'no', 'parking_space', 'parking_entrance', - 'waste_disposal', 'hunting_stand'}, - club = {'no'}, - craft = {'no'}, - leisure = {'no'}, - office = {'no'}, - mountain_pass = {'no'}, - shop = {'no'}, - tourism = {'yes', 'no'}, - bridge = {'no'}, - tunnel = {'no'}, - waterway = {'riverbank'}, - building = {'no'}, - boundary = {'place', 'land_area'}}, - extra_keys = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'wikidata', '*:wikidata', - '*:wikipedia', 'brand:wikipedia:*', - 'addr:street:name', 'addr:street:type'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*', 'brand'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}, - house = {'addr:housename'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'} - } - - -flex.set_unused_handling{delete_keys = {'tiger:*'}} +flex.load_topic('full', {with_extratags = true}) return flex diff --git a/lib-lua/import-full.lua b/lib-lua/import-full.lua index 11bd1f3a7..59308a672 100644 --- a/lib-lua/import-full.lua +++ b/lib-lua/import-full.lua @@ -1,126 +1,6 @@ +-- This is just an alias for the Nominatim themepark full topic local flex = require('flex-base') -flex.set_main_tags{ - building = 'fallback', - emergency = 'always', - healthcare = 'fallback', - historic = 'always', - military = 'always', - natural = 'named', - highway = {'always', - street_lamp = 'named', - traffic_signals = 'named', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - railway = 'named', - man_made = {'none', - pier = 'always', - tower = 'always', - bridge = 'always', - works = 'named', - water_tower = 'always', - dyke = 'named', - adit = 'named', - lighthouse = 'always', - watermill = 'always', - tunnel = 'always'}, - aerialway = 'always', - boundary = {'named', - postal_code = 'always'}, - aeroway = 'always', - amenity = 'always', - club = 'always', - craft = 'always', - junction = 'fallback', - landuse = 'fallback', - leisure = {'always', - nature_reserve = 'fallback'}, - office = 'always', - mountain_pass = 'always', - shop = 'always', - tourism = 'always', - bridge = 'named_with_key', - tunnel = 'named_with_key', - waterway = 'named', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'note', 'note:*', 'source', '*source', 'attribution', - 'comment', 'fixme', 'FIXME', 'created_by', 'NHD:*', - 'nhd:*', 'gnis:*', 'geobase:*', 'KSJ2:*', 'yh:*', - 'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*', - 'type', 'lacounty:*', 'ref:ruian:*', 'building:ruian:type', - 'ref:linz:*', 'is_in:postcode', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - '*:wikidata', '*:wikipedia', 'brand:wikipedia:*', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {emergency = {'yes', 'no', 'fire_hydrant'}, - historic = {'yes', 'no'}, - military = {'yes', 'no'}, - natural = {'yes', 'no', 'coastline'}, - highway = {'no', 'turning_circle', 'mini_roundabout', - 'noexit', 'crossing', 'give_way', 'stop'}, - railway = {'level_crossing', 'no', 'rail', 'switch', - 'abandoned', 'signal', 'buffer_stop', 'razed'}, - aerialway = {'pylon', 'no'}, - aeroway = {'no'}, - amenity = {'no', 'parking_space', 'parking_entrance', - 'waste_disposal', 'hunting_stand'}, - club = {'no'}, - craft = {'no'}, - leisure = {'no'}, - office = {'no'}, - mountain_pass = {'no'}, - shop = {'no'}, - tourism = {'yes', 'no'}, - bridge = {'no'}, - tunnel = {'no'}, - waterway = {'riverbank'}, - building = {'no'}, - boundary = {'place', 'land_area'}}, - extra_keys = {'wikidata', 'wikipedia', 'wikipedia:*'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*', 'brand'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'}, - house = {'addr:housename'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'} - } - - -flex.set_unused_handling{extra_keys = {'place'}} +flex.load_topic('full') return flex diff --git a/lib-lua/import-street.lua b/lib-lua/import-street.lua index 60b76dfb3..1a02b0985 100644 --- a/lib-lua/import-street.lua +++ b/lib-lua/import-street.lua @@ -1,74 +1,6 @@ +-- This is just an alias for the Nominatim themepark street topic local flex = require('flex-base') -flex.set_main_tags{ - highway = {motorway = 'always', - trunk = 'always', - primary = 'always', - secondary = 'always', - tertiary = 'always', - unclassified = 'always', - residential = 'always', - road = 'always', - living_street = 'always', - pedestrian = 'always', - service = 'named', - cycleway = 'named', - path = 'named', - footway = 'named', - steps = 'named', - bridleway = 'named', - track = 'named', - motorway_link = 'named', - trunk_link = 'named', - primary_link = 'named', - secondary_link = 'named', - tertiary_link = 'named'}, - boundary = {administrative = 'named', - postal_code = 'always'}, - landuse = 'fallback', - place = 'always' -} - -flex.set_prefilters{delete_keys = {'building', 'source', - 'addr:housenumber', 'addr:street', - 'source', '*source', 'type', - 'is_in:postcode', '*:wikidata', '*:wikipedia', - '*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical', - 'addr:street:name', 'addr:street:type'}, - delete_tags = {landuse = {'cemetry', 'no'}, - boundary = {'place'}}, - extra_keys = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital', 'area'} - } - -flex.set_name_tags{main = {'name', 'name:*', - 'int_name', 'int_name:*', - 'nat_name', 'nat_name:*', - 'reg_name', 'reg_name:*', - 'loc_name', 'loc_name:*', - 'old_name', 'old_name:*', - 'alt_name', 'alt_name:*', 'alt_name_*', - 'official_name', 'official_name:*', - 'place_name', 'place_name:*', - 'short_name', 'short_name:*'}, - extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', 'pcode', 'pcode:*', 'ISO3166-2'} - } - -flex.set_address_tags{main = {'addr:housenumber', - 'addr:conscriptionnumber', - 'addr:streetnumber'}, - extra = {'addr:*', 'is_in:*', 'tiger:county'}, - postcode = {'postal_code', 'postcode', 'addr:postcode', - 'tiger:zip_left', 'tiger:zip_right'}, - country = {'country_code', 'ISO3166-1', - 'addr:country_code', 'is_in:country_code', - 'addr:country', 'is_in:country'}, - interpolation = {'addr:interpolation'}, - postcode_fallback = false - } - -flex.set_unused_handling{extra_keys = {'place'}} +flex.load_topic('street') return flex diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index f8706bb92..8f88f0c1a 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -65,6 +65,8 @@ local place_table_definition = { } local insert_row +local script_path = debug.getinfo(1, "S").source:match("@?(.*/)") +local PRESETS = loadfile(script_path .. 'presets.lua')() if themepark then themepark:add_table(place_table_definition) @@ -687,12 +689,26 @@ end function module.ignore_keys(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.IGNORE_KEYS[data] + if data == nil then + error('Unknown preset for ignored keys: ' .. preset) + end + end merge_filters_into_main('delete', data) add_pre_filter{delete = data} end function module.add_for_extratags(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.EXTRATAGS[data] or PRESETS.IGNORE_KEYS[data] + if data == nil then + error('Unknown preset for extratags: ' .. preset) + end + end merge_filters_into_main('extra', data) add_pre_filter{extra = data} end @@ -709,11 +725,25 @@ function module.set_main_tags(data) MAIN_KEYS[key] = nil end end - module.add_main_tags(data) + module.modify_main_tags(data) end -function module.add_main_tags(data) +function module.modify_main_tags(data) + if type(data) == 'string' then + local preset = data + if data:sub(1, 7) == 'street/' then + data = PRESETS.MAIN_TAGS_STREETS[data:sub(8)] + elseif data:sub(1, 4) == 'poi/' then + data = PRESETS.MAIN_TAGS_POIS(data:sub(5)) + else + data = PRESETS.MAIN_TAGS[data] + end + if data == nil then + error('Unknown preset for main tags: ' .. preset) + end + end + for k, v in pairs(data) do if MAIN_KEYS[k] == nil then MAIN_KEYS[k] = {} @@ -736,6 +766,14 @@ end function module.modify_name_tags(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.NAME_TAGS[data] + if data == nil then + error('Unknown preset for name keys: ' .. preset) + end + end + for k,v in pairs(data) do if next(v) then NAMES[k] = v @@ -764,6 +802,14 @@ end function module.modify_address_tags(data) + if type(data) == 'string' then + local preset = data + data = PRESETS.ADDRESS_TAGS[data] + if data == nil then + error('Unknown preset for address keys: ' .. preset) + end + end + for k, v in pairs(data) do if k == 'postcode_fallback' then POSTCODE_FALLBACK = v diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua new file mode 100644 index 000000000..019328125 --- /dev/null +++ b/lib-lua/themes/nominatim/presets.lua @@ -0,0 +1,281 @@ +-- Defines defaults used in the topic definitions. + +local module = {} + +-- Main tag definition + +module.MAIN_TAGS = {} + +module.MAIN_TAGS.admin = { + boundary = {administrative = 'named'}, + landuse = {residential = 'fallback', + farm = 'fallback', + farmyard = 'fallback', + industrial = 'fallback', + commercial = 'fallback', + allotments = 'fallback', + retail = 'fallback'}, + place = {county = 'always', + district = 'always', + municipality = 'always', + city = 'always', + town = 'always', + borough = 'always', + village = 'always', + suburb = 'always', + hamlet = 'always', + croft = 'always', + subdivision = 'always', + allotments = 'always', + neighbourhood = 'always', + quarter = 'always', + isolated_dwelling = 'always', + farm = 'always', + city_block = 'always', + locality = 'always'} +} + +module.MAIN_TAGS.all_boundaries = { + boundary = {'named', + place = 'delete', + land_area = 'delete', + postal_code = 'always'}, + landuse = 'fallback', + place = 'always' +} + +module.MAIN_TAGS.natural = { + waterway = {'named', + riverbank = 'delete'}, + natural = {'named', + yes = 'delete', + no = 'delete', + coastline = 'delete', + saddle = 'fallback'}, + mountain_pass = {'always', + no = 'delete'} +} + +module.MAIN_TAGS_POIS = function (group) + group = group or 'delete' + return { + aerialway = {'always', + no = group, + pylon = group}, + aeroway = {'always', + no = group}, + amenity = {'always', + no = group, + parking_space = group, + parking_entrance = group, + waste_disposal = group, + hunting_stand = group}, + building = {'fallback', + no = group}, + bridge = {'named_with_key', + no = group}, + club = {'always', + no = group}, + craft = {'always', + no = group}, + emergency = {'always', + no = group, + yes = group, + fire_hydrant = group}, + healthcare = {'fallback', + yes = group, + no = group}, + highway = {'always', + no = group, + turning_circle = group, + mini_roundabout = group, + noexit = group, + crossing = group, + give_way = group, + stop = group, + turning_loop = group, + passing_place = group, + street_lamp = 'named', + traffic_signals = 'named'}, + historic = {'always', + yes = group, + no = group}, + junction = {'fallback', + no = group}, + leisure = {'always', + nature_reserve = 'fallback', + no = group}, + man_made = {pier = 'always', + tower = 'always', + bridge = 'always', + works = 'named', + water_tower = 'always', + dyke = 'named', + adit = 'named', + lighthouse = 'always', + watermill = 'always', + tunnel = 'always'}, + military = {'always', + yes = group, + no = group}, + office = {'always', + no = group}, + railway = {'named', + rail = group, + no = group, + abandoned = group, + disused = group, + razed = group, + level_crossing = group, + switch = group, + signal = group, + buffer_stop = group}, + shop = {'always', + no = group}, + tourism = {'always', + no = group, + yes = group}, + tunnel = {'named_with_key', + no = group} +} end + +module.MAIN_TAGS_STREETS = {} + +module.MAIN_TAGS_STREETS.default = { + place = {square = 'always'}, + highway = {motorway = 'always', + trunk = 'always', + primary = 'always', + secondary = 'always', + tertiary = 'always', + unclassified = 'always', + residential = 'always', + road = 'always', + living_street = 'always', + pedestrian = 'always', + service = 'named', + cycleway = 'named', + path = 'named', + footway = 'named', + steps = 'named', + bridleway = 'named', + track = 'named', + motorway_link = 'named', + trunk_link = 'named', + primary_link = 'named', + secondary_link = 'named', + tertiary_link = 'named'} +} + +module.MAIN_TAGS_STREETS.car = { + place = {square = 'always'}, + highway = {motorway = 'always', + trunk = 'always', + primary = 'always', + secondary = 'always', + tertiary = 'always', + unclassified = 'always', + residential = 'always', + road = 'always', + living_street = 'always', + service = 'always', + track = 'always', + motorway_link = 'always', + trunk_link = 'always', + primary_link = 'always', + secondary_link = 'always', + tertiary_link = 'always'} +} + +module.MAIN_TAGS_STREETS.all = { + place = {square = 'always'}, + highway = {motorway = 'always', + trunk = 'always', + primary = 'always', + secondary = 'always', + tertiary = 'always', + unclassified = 'always', + residential = 'always', + road = 'always', + living_street = 'always', + pedestrian = 'always', + service = 'always', + cycleway = 'always', + path = 'always', + footway = 'always', + steps = 'always', + bridleway = 'always', + track = 'always', + motorway_link = 'always', + trunk_link = 'always', + primary_link = 'always', + secondary_link = 'always', + tertiary_link = 'always'} +} + + +-- name tags + +module.NAME_TAGS = {} + +module.NAME_TAGS.core = {main = {'name', 'name:*', + 'int_name', 'int_name:*', + 'nat_name', 'nat_name:*', + 'reg_name', 'reg_name:*', + 'loc_name', 'loc_name:*', + 'old_name', 'old_name:*', + 'alt_name', 'alt_name:*', 'alt_name_*', + 'official_name', 'official_name:*', + 'place_name', 'place_name:*', + 'short_name', 'short_name:*'}, + extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', + 'loc_ref', 'old_ref', 'ISO3166-2'} + } +module.NAME_TAGS.address = {house = {'addr:housename'}} +module.NAME_TAGS.poi = {extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', + 'loc_ref', 'old_ref', + 'iata', 'icao', + 'ISO3166-2'}} + +-- Address tagging + +module.ADDRESS_TAGS = {} + +module.ADDRESS_TAGS.core = { extra = {'addr:*', 'is_in:*', 'tiger:county'}, + postcode = {'postal_code', 'postcode', 'addr:postcode', + 'tiger:zip_left', 'tiger:zip_right'}, + country = {'country_code', 'ISO3166-1', + 'addr:country_code', 'is_in:country_code', + 'addr:country', 'is_in:country'} + } + +module.ADDRESS_TAGS.houses = { main = {'addr:housenumber', + 'addr:conscriptionnumber', + 'addr:streetnumber'}, + interpolation = {'addr:interpolation'} + } + +-- Ignored tags (prefiltered away) + +module.IGNORE_KEYS = {} + +module.IGNORE_KEYS.metatags = {'note', 'note:*', 'source', 'source:*', '*source', + 'attribution', 'comment', 'fixme', 'created_by', + 'tiger:cfcc', 'tiger:reviewed', 'nysgissam:*', + 'NHD:*', 'nhd:*', 'gnis:*', 'geobase:*', 'yh:*', + 'osak:*', 'naptan:*', 'CLC:*', 'import', 'it:fvg:*', + 'lacounty:*', 'ref:linz:*', + 'ref:bygningsnr', 'ref:ruian:*', 'building:ruian:type', + 'type', + 'is_in:postcode'} +module.IGNORE_KEYS.name = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', + 'name:etymology', 'name:signed', 'name:botanical'} +module.IGNORE_KEYS.address = {'addr:street:*', 'addr:TW:dataset'} + +-- Extra tags (prefiltered away) + +module.EXTRATAGS = {} + +module.EXTRATAGS.required = {'wikipedia', 'wikipedia:*', 'wikidata', 'capital'} + +return module diff --git a/lib-lua/themes/nominatim/topics/address.lua b/lib-lua/themes/nominatim/topics/address.lua new file mode 100644 index 000000000..0e813673a --- /dev/null +++ b/lib-lua/themes/nominatim/topics/address.lua @@ -0,0 +1,23 @@ +local _, flex, cfg = ... + +flex.set_main_tags('admin') +flex.modify_main_tags('street/' .. (cfg.street_theme or 'default')) +flex.modify_main_tags{boundary = {postal_code = 'always'}} + +flex.set_name_tags('core') +flex.modify_name_tags('address') + +flex.set_address_tags('core') +flex.modify_address_tags('houses') + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/lib-lua/themes/nominatim/topics/admin.lua b/lib-lua/themes/nominatim/topics/admin.lua new file mode 100644 index 000000000..47f4e248d --- /dev/null +++ b/lib-lua/themes/nominatim/topics/admin.lua @@ -0,0 +1,20 @@ +local _, flex, cfg = ... + +flex.set_main_tags('admin') + +flex.set_name_tags('core') + +flex.set_address_tags('core') +flex.set_postcode_fallback(false) + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/lib-lua/themes/nominatim/topics/full.lua b/lib-lua/themes/nominatim/topics/full.lua new file mode 100644 index 000000000..a0b61b0f8 --- /dev/null +++ b/lib-lua/themes/nominatim/topics/full.lua @@ -0,0 +1,32 @@ +local _, flex, cfg = ... + +local group +if cfg.with_extratags then + group = 'extra' +else + group = 'delete' +end + +flex.set_main_tags('all_boundaries') +flex.modify_main_tags('natural') +flex.modify_main_tags('street/' .. (cfg.street_theme or 'default')) +flex.modify_main_tags('poi/' .. group) + +flex.set_name_tags('core') +flex.modify_name_tags('address') +flex.modify_name_tags('poi') + +flex.set_address_tags('core') +flex.modify_address_tags('houses') + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/lib-lua/themes/nominatim/topics/street.lua b/lib-lua/themes/nominatim/topics/street.lua new file mode 100644 index 000000000..89bed4426 --- /dev/null +++ b/lib-lua/themes/nominatim/topics/street.lua @@ -0,0 +1,22 @@ +local _, flex, cfg = ... + +flex.set_main_tags('admin') +flex.modify_main_tags('street/' .. (cfg.street_theme or 'default')) +flex.modify_main_tags{boundary = {postal_code = 'always'}} + +flex.set_name_tags('core') + +flex.set_address_tags('core') +flex.set_postcode_fallback(false) + +flex.ignore_keys('metatags') +flex.add_for_extratags('required') + +if cfg.with_extratags then + flex.set_unused_handling{delete_keys = {'tiger:*'}} + flex.add_for_extratags('name') + flex.add_for_extratags('address') +else + flex.ignore_keys('name') + flex.ignore_keys('address') +end diff --git a/test/bdd/api/details/simple.feature b/test/bdd/api/details/simple.feature index 5e0bacc5a..a3cc95e53 100644 --- a/test/bdd/api/details/simple.feature +++ b/test/bdd/api/details/simple.feature @@ -27,7 +27,6 @@ Feature: Object details Examples: | class | | tourism | - | natural | | mountain_pass | diff --git a/test/bdd/osm2pgsql/import/custom_style.feature b/test/bdd/osm2pgsql/import/custom_style.feature index 405afdb5c..15852c5d3 100644 --- a/test/bdd/osm2pgsql/import/custom_style.feature +++ b/test/bdd/osm2pgsql/import/custom_style.feature @@ -33,7 +33,7 @@ Feature: Import with custom styles by osm2pgsql """ local flex = require('import-full') - flex.add_main_tags{ + flex.modify_main_tags{ amenity = {prison = 'delete'}, highway = {stop = 'named'}, aeroway = 'named' diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index 7958f4b35..cf530eb7d 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -131,7 +131,7 @@ Feature: Tag evaluation When loading osm data """ n8001 Tshop=shoes,note:de=Nein,xx=yy - n8002 Tshop=shoes,building=no,ele=234 + n8002 Tshop=shoes,natural=no,ele=234 n8003 Tshop=shoes,name:source=survey """ Then place contains exactly From e1e8182c72d00fd1d66b38e7e49662ae0532fc15 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 6 Dec 2024 15:11:16 +0100 Subject: [PATCH 08/17] adapt taginfo script to new configuration structure --- lib-lua/taginfo.lua | 87 +++++++++++++++++++++++-------- lib-lua/themes/nominatim/init.lua | 11 ++-- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/lib-lua/taginfo.lua b/lib-lua/taginfo.lua index fddaf298f..402499ad2 100644 --- a/lib-lua/taginfo.lua +++ b/lib-lua/taginfo.lua @@ -7,13 +7,31 @@ function osm2pgsql.define_table(...) end -- provide path to flex-style lua file package.path = arg[0]:match("(.*/)") .. "?.lua;" .. package.path -local flex = require('import-extratags') +local flex = require('import-' .. (arg[1] or 'extratags')) local json = require ('dkjson') +local NAME_DESCRIPTIONS = { + 'Searchable auxiliary name of the place', + main = 'Searchable primary name of the place', + house = 'House name part of an address, searchable' +} +local ADDRESS_DESCRIPTIONS = { + 'Used to determine the address of a place', + main = 'Primary key for an address point', + postcode = 'Used to determine the postcode of a place', + country = 'Used to determine country of a place (only if written as two-letter code)', + interpolation = 'Primary key for an address interpolation line' +} ------------ helper functions --------------------- +-- Sets the key order for the resulting JSON table +local function set_keyorder(table, order) + setmetatable(table, { + __jsonorder = order + }) +end -function get_key_description(key, description) +local function get_key_description(key, description) local desc = {} desc.key = key desc.description = description @@ -21,35 +39,60 @@ function get_key_description(key, description) return desc end --- Sets the key order for the resulting JSON table -function set_keyorder(table, order) - setmetatable(table, { - __jsonorder = order - }) +local function get_key_value_description(key, value, description) + local desc = {key = key, value = value, description = description} + set_keyorder(desc, {'key', 'value', 'description'}) + return desc end +local function group_table_to_keys(tags, data, descriptions) + for group, values in pairs(data) do + local desc = descriptions[group] or descriptions[1] + for _, key in pairs(values) do + if key:sub(1, 1) ~= '*' and key:sub(#key, #key) ~= '*' then + table.insert(tags, get_key_description(key, desc)) + end + end + end +end -- Prints the collected tags in the required format in JSON -function print_taginfo() +local function print_taginfo() + local taginfo = flex.get_taginfo() local tags = {} - for _, k in ipairs(flex.TAGINFO_MAIN.keys) do - local desc = get_key_description(k, 'POI/feature in the search database') - if flex.TAGINFO_MAIN.delete_tags[k] ~= nil then - desc.description = string.format('%s (except for values: %s).', desc.description, - table.concat(flex.TAGINFO_MAIN.delete_tags[k], ', ')) + for k, values in pairs(taginfo.main) do + if values[1] == nil or values[1] == 'delete' or values[1] == 'extra' then + for v, group in pairs(values) do + if type(v) == 'string' and group ~= 'delete' and group ~= 'extra' then + local text = 'POI/feature in the search database' + if type(group) ~= 'function' then + text = 'Fallback ' .. text + end + table.insert(tags, get_key_value_description(k, v, text)) + end + end + elseif type(values[1]) == 'function' or values[1] == 'fallback' then + local desc = 'POI/feature in the search database' + if values[1] == 'fallback' then + desc = 'Fallback ' .. desc + end + local excp = {} + for v, group in pairs(values) do + if group == 'delete' or group == 'extra' then + table.insert(excp, v) + end + end + if next(excp) ~= nil then + desc = desc .. string.format(' (except for values: %s)', + table.concat(excp, ', ')) + end + table.insert(tags, get_key_description(k, desc)) end - table.insert(tags, desc) end - for k, _ in pairs(flex.TAGINFO_NAME_KEYS) do - local desc = get_key_description(k, 'Searchable name of the place.') - table.insert(tags, desc) - end - for k, _ in pairs(flex.TAGINFO_ADDRESS_KEYS) do - local desc = get_key_description(k, 'Used to determine the address of a place.') - table.insert(tags, desc) - end + group_table_to_keys(tags, taginfo.name, NAME_DESCRIPTIONS) + group_table_to_keys(tags, taginfo.address, ADDRESS_DESCRIPTIONS) local format = { data_format = 1, diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 8f88f0c1a..6d3804e27 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -40,12 +40,6 @@ if type(themepark) ~= 'table' then themepark = nil end --- tables required for taginfo -module.TAGINFO_MAIN = {keys = {}, delete_tags = {}} -module.TAGINFO_NAME_KEYS = {} -module.TAGINFO_ADDRESS_KEYS = {} - - -- The single place table. local place_table_definition = { name = "place", @@ -904,4 +898,9 @@ function module.set_relation_types(data) end end + +function module.get_taginfo() + return {main = MAIN_KEYS, name = NAMES, address = ADDRESS_TAGS} +end + return module From a75dd32f75c9ae8cb25c2d5038c2fb6392cf450a Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 9 Dec 2024 12:00:22 +0100 Subject: [PATCH 09/17] adapt documentation for style import --- docs/customize/Import-Styles.md | 583 ++++++++++++++++----------- docs/customize/Special-Phrases.md | 2 +- docs/extra.css | 4 +- lib-lua/themes/nominatim/presets.lua | 4 +- 4 files changed, 363 insertions(+), 230 deletions(-) diff --git a/docs/customize/Import-Styles.md b/docs/customize/Import-Styles.md index 003e56e38..59589a0ac 100644 --- a/docs/customize/Import-Styles.md +++ b/docs/customize/Import-Styles.md @@ -1,95 +1,123 @@ -## Configuring the Import +# Configuring the Import of OSM data In the very first step of a Nominatim import, OSM data is loaded into the database. Nominatim uses [osm2pgsql](https://osm2pgsql.org) for this task. It comes with a [flex style](https://osm2pgsql.org/doc/manual.html#the-flex-output) specifically tailored to filter and convert OSM data into Nominatim's -internal data representation. - -There are a number of default configurations for the flex style which -result in geocoding databases of different detail. The +internal data representation. Nominatim ships with a few preset +configurations for this import, each results in a geocoding database of +different detail. The [Import section](../admin/Import.md#filtering-imported-data) explains these default configurations in detail. -You can also create your own custom style. Put the style file into your -project directory and then set `NOMINATIM_IMPORT_STYLE` to the name of the file. -It is always recommended to start with one of the standard styles and customize -those. You find the standard styles under the name `import-.lua` -in the standard Nominatim configuration path (usually `/etc/nominatim` or -`/usr/local/etc/nominatim`). +If you want to have more control over which OSM data is added to the database, +you can also create your own custom style. Create a new lua style file, put it +into your project directory and then set `NOMINATIM_IMPORT_STYLE` to the name +of the file. Custom style files can be used to modify the existing preset +configurations or to implement your own configuration from scratch. The remainder of the page describes how the flex style works and how to customize it. -### The `flex-base.lua` module +## The `flex-base` lua module The core of Nominatim's flex import configuration is the `flex-base` module. It defines the table layout used by Nominatim and provides standard -implementations for the import callbacks that make it easy to customize +implementations for the import callbacks that help with customizing how OSM tags are used by Nominatim. -Every custom style should include this module to make sure that the correct +Every custom style must include this module to make sure that the correct tables are created. Thus start your custom style as follows: ``` lua local flex = require('flex-base') +``` + +### Using preset configurations + +If you want to start with one of the existing presets, then you can import +its settings using the `import_topic()` function: + +``` +local flex = require('flex-base') +flex.import_topic('streets') ``` -The following sections explain how the module can be customized. +The `import_topic` function takes an optional second configuration +parameter. The available options are explained in the +[themepark section](#using-osm2pgsql-themepark). +!!! note + You can also directly import the preset style files, e.g. + `local flex = require('import-street')`. It is not possible to + set extra configuration this way. -### Changing the recognized tags +### How processing works -If you just want to change which OSM tags are recognized during import, -then there are a number of convenience functions to set the tag lists used -during the processing. +When Nominatim processes an OSM object, it looks for four kinds of tags: +The _main tags_ classify what kind of place the OSM object represents. One +OSM object can have more than one main tag. In such case one database entry +is created for each main tag. _Name tags_ represent searchable names of the +place. _Address tags_ are used to compute the address hierarchy of the place. +Address are used for searching and for creating a display name of the place. +_Extra tags_ are any tags that are not directly related to search but +contain interesting additional information. -!!! warning - There are no built-in defaults for the tag lists, so all the functions - need to be called from your style script to fully process the data. - Make sure you start from one of the default style and only modify - the data you are interested in. You can also derive your style from an - existing style by importing the appropriate module, e.g. - `local flex = require('import-street')`. +!!! danger + Some tags in the extratags category are used by Nominatim to better + classify the place. You want to make sure these are always present + in custom styles. -Many of the following functions take _key match lists_. These lists can +Configuring the style means deciding which key and/or key/value is used +in which category. + +## Changing the recognized tags + +The flex style offers a number of functions to set the classification of +each OSM tag. Most of these functions can also take a preset string instead +of a tag descriptions. These presets describe common configurations that +are also used in the definition of the predefined styles. This section +lists the configuration functions and the accepted presets. + +#### Key match lists + +Some of the following functions take _key match lists_. These lists can contain three kinds of strings to match against tag keys: A string that ends in an asterisk `*` is a prefix match and accordingly matches against any key that starts with the given string (minus the `*`). A suffix match can be defined similarly with a string that starts with a `*`. Any other string is matched exactly against tag keys. +### Main tags + +`set/modify_main_tags()` allow to define which tags are used as main tags. It +takes a lua table parameter which defines for keys and key/value +combinations, how they are classified. + +The following classifications are recognised: -#### `set_main_tags()` - principal tags - -If a principal or main tag is found on an OSM object, then the object -is included in Nominatim's search index. A single object may also have -multiple main tags. In that case, the object will be included multiple -times in the index, once for each main tag. - -The flex script distinguishes between four types of main tags: - -* __always__: a main tag that is used unconditionally -* __named__: consider this main tag only, if the object has a proper name - (a reference is not enough, see below). -* __named_with_key__: consider this main tag only, when the object has - a proper name with a domain prefix. For example, if the main tag is - `bridge=yes`, then it will only be added as an extra row, if there is - a tag `bridge:name[:XXX]` for the same object. If this property is set, - all other names that are not domain-specific are ignored. -* __fallback__: use this main tag only, if there is no other main tag. - Fallback always implied `named`, i.e. fallbacks are only tried for - named objects. - -The `set_main_tags()` function takes exactly one table parameter which -defines the keys and key/value combinations to include and the kind of -main tag. Each lua table key defines an OSM tag key. The value may -be a string defining the kind of main key as described above. Then the tag will -be considered a main tag for any possible value. To further restrict -which values are acceptable, give a table with the permitted values -and their kind of main tag. If the table contains a simple value without -key, then this is used as default for values that are not listed. +| classification | meaning | +| :-------------- | :------ | +| always | Unconditionally use this tag as a main tag. | +| named | Consider as main tag, when the object has a primary name (see [names](#name-tags) below) | +| named_with_key | Consider as main tag, when the object has a primary name with a domain prefix. For example, if the main tag is `bridge=yes`, then it will only be added as an extra entry, if there is a tag `bridge:name[:XXX]` for the same object. If this property is set, all names that are not domain-specific are ignored. | +| fallback | Consider as main tag only when no other main tag was found. Fallback always implies `named`, i.e. fallbacks are only tried for objects with primary names. | +| delete | Completely ignore the tag in any further processing | +| extra | Move the tag to extratags and then ignore it for further processing | +| ``| Advanced handling, see [below](#advanced-main-tag-handling) | + +Each key in the table parameter defines an OSM tag key. The value may +be directly a classification as described above. Then the tag will +be considered a main tag for any possible value that is not further defined. +To further restrict which values are acceptable, give a table with the +permitted values and their kind of main tag. If the table contains a simple +value without key, then this is used as default for values that are not listed. + +`set_main_tags()` will completely replace the current main tag configuration +with the new configuration. `modify_main_tags()` will merge the new +configuration with the existing one. Otherwise, the two functions do exactly +the same. !!! example ``` lua @@ -97,80 +125,188 @@ key, then this is used as default for values that are not listed. flex.set_main_tags{ boundary = {administrative = 'named'}, - highway = {'always', street_lamp = 'named'}, + highway = {'always', street_lamp = 'named', no = 'delete'}, landuse = 'fallback' } ``` In this example an object with a `boundary` tag will only be included when it has a value of `administrative`. Objects with `highway` tags are - always included. However when the value is `street_lamp` then the object - must have a name, too. With any other value, the object is included - independently of the name. Finally, if a `landuse` tag is present then - it will be used independely of the concrete value if neither boundary + always included with two exceptions: the troll tag `highway=no` is + deleted on the spot and when the value is `street_lamp` then the object + must have a name, too. Finally, if a `landuse` tag is present then + it will be used independently of the concrete value when neither boundary nor highway tags were found and the object is named. +##### Presets -#### `set_prefilters()` - ignoring tags +| Name | Description | +| :----- | :---------- | +| admin | Basic tag set collecting places and administrative boundaries. This set is needed also to ensure proper address computation and should therefore always be present. You can disable selected place types like `place=locality` after adding this set, if they are not relevant for your use case. | +| all_boundaries | Extends the set of recognised boundaries and places to all available ones. | +| natural | Tags for natural features like rivers and mountain peaks. | +| street/default | Tags for streets. Major streets are always included, minor ones only when they have a name. | +| street/car | Tags for all streets that can be used by a motor vehicle. | +| street/all | Includes all highway features named and unnamed. | +| poi/delete | Adds most POI features with and without name. Some frequent but very domain-specific values are excluded by deleting them. | +| poi/extra | Like 'poi/delete' but excluded values are moved to extratags. | -Pre-filtering of tags allows to ignore them for any further processing. -Thus pre-filtering takes precedence over any other tag processing. This is -useful when some specific key/value combinations need to be excluded from -processing. When tags are filtered, they may either be deleted completely -or moved to `extratags`. Extra tags are saved with the object and returned -to the user when requested, but are not used otherwise. -`set_prefilters()` takes a table with four optional fields: +##### Advanced main tag handling -* __delete_keys__ is a _key match list_ for tags that should be deleted -* __delete_tags__ contains a table of tag keys pointing to a list of tag - values. Tags with matching key/value pairs are deleted. -* __extra_keys__ is a _key match list_ for tags which should be saved into - extratags -* __extra_tags__ contains a table of tag keys pointing to a list of tag - values. Tags with matching key/value pairs are moved to extratags. +The groups described above are in fact only a preset for a filtering function +that is used to make the final decision how a pre-selected main tag is entered +into Nominatim's internal table. To further customize handling you may also +supply your own filtering function. -Key list may contain three kinds of strings: -A string that ends in an asterisk `*` is a prefix match and accordingly matches -against any key that starts with the given string (minus the `*`). -A suffix match can be defined similarly with a string that starts with a `*`. -Any other string is matched exactly against tag keys. +The function takes up to three parameters: a Place object of the object +being processed, the key of the main tag and the value of the main tag. +The function may return one of three values: + +* `nil` or `false` causes the entry to be ignored +* the Place object causes the place to be added as is +* `Place.copy(names=..., address=..., extratags=...) causes the + place to be enter into the database but with name/address/extratags + set to the given different values. + +The Place object has some read-only values that can be used to determine +the handling: + +* **object** is the original OSM object data handed in by osm2pgsql +* **admin_level** is the content of the admin_level tag, parsed into an integer and normalized to a value between 0 and 15 +* **has_name** is a boolean indicating if the object has a primary name tag +* **names** is a table with the collected list of name tags +* **address** is a table with the collected list of address tags +* **extratags** is a table with the collected list of additional tags to save !!! example ``` lua - local flex = require('import-full') + local flex = require('flex-base') - flex.set_prefilters{ - delete_keys = {'source', 'source:*'}, - extra_tags = {amenity = {'yes', 'no'}} - } - flex.set_main_tags{ - amenity = 'always' - } + flex.add_topic('street') + + local function no_sidewalks(place, k, v) + if place.object.tags.footway == 'sidewalk' then + return false + end + + -- default behaviour is to have all footways + return place + end + + flex.modify_main_tags(highway = {'footway' = no_sidewalks} + ``` + This script adds a custom handler for `highway=footway`. It only includes + them in the database, when the object doesn't have a tag `footway=sidewalk` + indicating that it is just part of a larger street which should already + be indexed. Note that it is not necessary to check the key and value + of the main tag because the function is only used for the specific + main tag. + + +### Ignored tags + +The function `ignore_keys()` sets the `delete` classification for keys. +This function takes a _key match list_ so that it is possible to exclude +groups of keys. + +Note that full matches always take precedence over suffix matches, which +in turn take precedence over prefix matches. + +!!! example + ``` lua + local flex = require('flex-base') + + flex.add_topic('admin') + flex.ignore_keys{'old_name', 'old_name:*'} ``` - In this example any tags `source` and tags that begin with `source:` are - deleted before any other processing is done. Getting rid of frequent tags - this way can speed up the import. + This example uses the `admin` preset with the exception that names + that are no longer are in current use, are ignored. + +##### Presets - Tags with `amenity=yes` or `amenity=no` are moved to extratags. Later - all tags with an `amenity` key are made a main tag. This effectively means - that Nominatim will use all amenity tags except for those with value - yes and no. +| Name | Description | +| :----- | :---------- | +| metatags | Tags with meta information about the OSM tag like source, notes and import sources. | +| name | Non-names that describe in fact properties or name parts. These names can throw off search and should always be removed. | +| address | Extra `addr:*` tags that are not useful for Nominatim. | -#### `set_name_tags()` - defining names -The flex script distinguishes between two kinds of names: +### Tags for `extratags` -* __main__: the primary names make an object fully searchable. - Main tags of type _named_ will only cause the object to be included when - such a primary name is present. Primary names are usually those found - in the `name` tag and its variants. -* __extra__: extra names are still added to the search index but they are - alone not sufficient to make an object named. +The function `add_for_extratags()` sets the `extra` classification for keys. +This function takes a +_key match list_ so that it is possible to move groups of keys to extratags. -`set_name_tags()` takes a table with two optional fields `main` and `extra`. -They take _key match lists_ for main and extra names respectively. +Note that full matches always take precedence over suffix matches, which +in turn take precedence over prefix matches. + +!!! example + ``` lua + local flex = require('flex-base') + + flex.add_topic('street') + flex.add_for_extratags{'surface', 'access', 'vehicle', 'maxspeed'} + ``` + + This example uses the `street` preset but adds a couple of tags that + are of interest about the condition of the street. + +##### Presets + +| Name | Description | +| :----- | :---------- | +| required | Tags that Nominatim will use for various computations when present in extratags. Always include these. | + +In addition, all [presets from ignored tags](#presets_1) are accepted. + +### General pre-filtering + +_(deprecated)_ `set_prefilters()` allows to set the `delete` and `extra` +classification for main tags. + +This function removes all previously set main tags with `delete` and `extra` +classification and then adds the newly defined tags. + +`set_prefilters()` takes a table with four optional fields: + +* __delete_keys__ is a _key match list_ for tags that should be deleted +* __delete_tags__ contains a table of tag keys pointing to a list of tag + values. Tags with matching key/value pairs are deleted. +* __extra_keys__ is a _key match list_ for tags which should be saved into + extratags +* __extra_tags__ contains a table of tag keys pointing to a list of tag + values. Tags with matching key/value pairs are moved to extratags. + +!!! danger "Deprecation warning" + Use of this function should be replaced with `modify_main_tags()` to + set the data from `delete_tags` and `extra_tags`, with `ignore_keys()` + for the `delete_keys` parameter and with `add_for_extratags()` for the + `extra_keys` parameter. + +### Name tags + +`set/modify_name_tags()` allow to define the tags used for naming places. Name tags +can only be selected by their keys. The import script distinguishes +between primary and auxiliary names. A primary name is the given name of +a place. Having a primary name makes a place _named_. This is important +for main tags that are only included when a name is present. Auxiliary names +are identifiers like references. They may be searched for but should not +be included on their own. + +The functions take a table with two optional fields `main` and `extra`. +They take _key match lists_ for primary and auxiliary names respectively. +A third field `house` can contain tags for names that appear in place of +house numbers in addresses. This field can only contain complete key names. +'house tags' are special in that they cause the OSM object to be added to +the database independently of the presence of other main tags. + +`set_name_tags()` overwrites the current configuration, while +`modify_name_tags()` replaces the fields that are given. (Be aware that +the fields are replaced as a whole. `main = {'foo_name'}` will cause +`foo_name` to become the only recognised primary name. Any previously +defined primary names are forgotten.) !!! example ``` lua @@ -186,29 +322,33 @@ They take _key match lists_ for main and extra names respectively. only include those that have a common name and not those which just have some reference ID from the city. -#### `set_address_tags()` - defining address parts +##### Presets -Address tags will be used to build up the address of an object. +| Name | Description | +| :----- | :---------- | +| core | Basic set of recognised names for all places. | +| address | Additional names useful when indexing full addresses. | +| poi | Extended set of recognised names for pois. Use on top of the core set. | -`set_address_tags()` takes a table with arbitrary fields pointing to -_key match lists_. Two fields have a special meaning: +### Address tags -* __main__: defines -the tags that make a full address object out of the OSM object. This -is usually the housenumber or variants thereof. If a main address tag -appears, then the object will always be included, if necessary with a -fallback of `place=house`. If the key has a prefix of `addr:` or `is_in:` -this will be stripped. +`set/modify_address_tags()` defines the tags that will be used to build +up the address of an object. Address tags can only be chosen by their key. -* __extra__: defines all supplementary tags for addresses, tags like `addr:street`, `addr:city` etc. If the key has a prefix of `addr:` or `is_in:` this will be stripped. +The functions take a table with arbitrary fields, each defining +a key list or _key match list_. Some fields have a special meaning: -All other fields will be handled as summary fields. If a key matches the -key match list, then its value will be added to the address tags with the -name of the field as key. If multiple tags match, then an arbitrary one -wins. +| Field | Type | Description | +| :---------| :-------- | :-----------| +| main | key list | Tags that make a full address object out of the OSM object. This is usually the house number or variants thereof. If a main address tag appears, then the object will always be included, if necessary with a fallback of `place=house`. If the key has a prefix of `addr:` or `is_in:` this will be stripped. | +| extra | key match list | Supplementary tags for addresses, tags like `addr:street`, `addr:city` etc. If the key has a prefix of `addr:` or `is_in:` this will be stripped. | +| interpolation | key list | Tags that identify address interpolation lines. | +| country | key match list | Tags that may contain the country the place is in. The first found value with a two-letter code will be accepted, all other values are discarded. | +| _other_ | key match list | Summary field. If a key matches the key match list, then its value will be added to the address tags with the name of the field as key. If multiple tags match, then an arbitrary one wins. | -Country tags are handled slightly special. Only tags with a two-letter code -are accepted, all other values are discarded. +`set_address_tags()` overwrites the current configuration, while +`modify_address_tags()` replaces the fields that are given. (Be aware that +the fields are replaced as a whole.) !!! example ``` lua @@ -232,21 +372,33 @@ are accepted, all other values are discarded. to postcodes, they will always be saved under the key `postcode` thus normalizing the multitude of keys that are used in the OSM database. +##### Presets + +| Name | Description | +| :----- | :---------- | +| core | Basic set of tags needed to recognise address relationship for any place. Always include this. | +| houses | Additional set of tags needed to recognise proper addresses | -#### `set_unused_handling()` - processing remaining tags +### Handling of unclassified tags -This function defines what to do with tags that remain after all tags +`set_unused_handling()` defines what to do with tags that remain after all tags have been classified using the functions above. There are two ways in which the function can be used: `set_unused_handling(delete_keys = ..., delete_tags = ...)` deletes all keys that match the descriptions in the parameters and moves all remaining tags into the extratags list. + `set_unused_handling(extra_keys = ..., extra_tags = ...)` moves all tags matching the parameters into the extratags list and then deletes the remaining tags. For the format of the parameters see the description in `set_prefilters()` above. +When no special handling is set, then unused tags will be discarded with one +exception: place tags are kept in extratags for administrative boundaries. +When using a custom setting, you should also make sure that the place tag +is added for extratags. + !!! example ``` lua local flex = require('import-full') @@ -263,17 +415,23 @@ above. already delete the tiger tags with `set_prefilters()` because that would remove tiger:county before the address tags are processed. -### Customizing osm2pgsql callbacks +## Customizing osm2pgsql callbacks osm2pgsql expects the flex style to implement three callbacks, one process function per OSM type. If you want to implement special handling for certain OSM types, you can override the default implementations provided by the flex-base module. -#### Changing the relation types to be handled +### Enabling additional relation types -The default scripts only allows relations of type `multipolygon`, `boundary` -and `waterway`. To add other types relations, set `RELATION_TYPES` for +OSM relations can represent very diverse +[types of real-world objects](https://wiki.openstreetmap.org/wiki/Key:type). To +be able to process them correctly, Nominatim needs to understand how to +create a geometry for each type. By default, the script knows how to +process relations of type `multipolygon`, `boundary` and `waterway`. All +other relation types are ignored. + +To add other types relations, set `RELATION_TYPES` for the type to the kind of geometry that should be created. The following kinds of geometries can be used: @@ -297,7 +455,7 @@ kinds of geometries can be used: geometry. -#### Adding additional logic to processing functions +### Adding additional logic to processing functions The default processing functions are also exported by the flex-base module as `process_node`, `process_way` and `process_relation`. These can be used @@ -322,110 +480,83 @@ logic. ### Customizing the main processing function -The main processing function of the flex style can be found in the function -`process_tags`. This function is called for all OSM object kinds and is -responsible for filtering the tags and writing out the rows into Postgresql. +!!! danger "Deprecation Warning" + The style used to allow overwriting the internal processing function + `process_tags()`. While this is currently still possible, it is no longer + encouraged and may stop working in future versions. The internal + `Place` class should now be considered read-only. + + +## Using osm2pgsql-themepark + +The Nominatim osm2pgsql style is designed so that it can also be used as +a theme for [osm2pgsql-themepark](https://osm2pgsql.org/themepark/). This +makes it easy to combine Nominatim with other projects like +[openstreetmap-carto](https://github.com/gravitystorm/openstreetmap-carto) +in the same database. + +To set up one of the preset styles, simply include a topic with the same name: + +``` +local themepark = require('themepark') +themepark:add_topic('nominatim/address') +``` + +Themepark topics offer two configuration options: + +* **street_theme** allows to choose one of the sub topics for streets: + * _default_ - include all major streets and named minor paths + * _car_ - include all streets physically usable by cars + * _all_ - include all major streets and minor paths +* **with_extratags**, when set to a truthy value, then tags that are + not specifically used for address or naming are added to the + extratags column + +The customization functions described in the +[Changing recognized tags](#changing-the-recognized-tags) section +are available from the theme. To access the theme you need to explicitly initialise it. !!! Example ``` lua - local flex = require('import-full') + local themepark = require('themepark') - local original_process_tags = flex.process_tags + themepark:add_topic('nominatim/full', {with_extratags = true}) - function flex.process_tags(o) - if o.object.tags.highway ~= nil and o.object.tags.access == 'no' then - return - end + local flex = themepark:init_theme('nominatim') + + flex.modify_main_tags{'amenity' = { + 'waste_basket' = 'delete'} + } + ``` + This example uses the full Nominatim configuration but disables + importing waste baskets. + +You may also write a new configuration from scratch. Simply omit including +a Nominatim topic and only call the required customization functions. - original_process_tags(o) +Customizing the osm2pgsql processing functions as explained +[above](#adding-additional-logic-to-processing-functions) is not possible +when running under themepark. Instead include other topics that make the +necessary modifications or add an additional processor before including +the Nominatim topic. + +!!! Example + ``` lua + local themepark = require('themepark') + + local function discard_country_boundaries(object) + if object.tags.boundary == 'administrative' and object.tags.admin_level == '2' then + return 'stop' + end end + + themepark:add_proc('relation', discard_country_boundaries) + -- Order matters here. The topic needs to be added after the custom callback. + themepark:add_topic('nominatim/full', {with_extratags = true}) ``` + Discarding country-level boundaries when running under themepark. - This example shows the most simple customization of the process_tags function. - It simply adds some additional processing before running the original code. - To do that, first save the original function and then overwrite process_tags - from the module. In this example all highways which are not accessible - by anyone will be ignored. - - -#### The `Place` class - -The `process_tags` function receives a Lua object of `Place` type which comes -with some handy functions to collect the data necessary for geocoding and -writing it into the place table. Always use this object to fill the table. - -The Place class has some attributes which you may access read-only: - -* __object__ is the original OSM object data handed in by osm2pgsql -* __admin_level__ is the content of the admin_level tag, parsed into an - integer and normalized to a value between 0 and 15 -* __has_name__ is a boolean indicating if the object has a full name -* __names__ is a table with the collected list of name tags -* __address__ is a table with the collected list of address tags -* __extratags__ is a table with the collected list of additional tags to save - -There are a number of functions to fill these fields. All functions expect -a table parameter with fields as indicated in the description. -Many of these functions expect match functions which are described in detail -further below. - -* __delete{match=...}__ removes all tags that match the match function given - in _match_. -* __grab_extratags{match=...}__ moves all tags that match the match function - given in _match_ into extratags. Returns the number of tags moved. -* __clean{delete=..., extra=...}__ deletes all tags that match _delete_ and - moves the ones that match _extra_ into extratags -* __grab_address_parts{groups=...}__ moves matching tags into the address table. - _groups_ must be a group match function. Tags of the group `main` and - `extra` are added to the address table as is but with `addr:` and `is_in:` - prefixes removed from the tag key. All other groups are added with the - group name as key and the value from the tag. Multiple values of the same - group overwrite each other. The function returns the number of tags saved - from the main group. -* __grab_main_parts{groups=...}__ moves matching tags into the name table. - _groups_ must be a group match function. If a tags of the group `main` is - present, the object will be marked as having a name. Tags of group `house` - produce a fallback to `place=house`. This fallback is return by the function - if present. - -There are two functions to write a row into the place table. Both functions -expect the main tag (key and value) for the row and then use the collected -information from the name, address, extratags etc. fields to complete the row. -They also have a boolean parameter `save_extra_mains` which defines how any -unprocessed tags are handled: when True, the tags will be saved as extratags, -when False, they will be simply discarded. - -* __write_row(key, value, save_extra_mains)__ creates a new table row from - the current state of the Place object. -* __write_place(key, value, mtype, save_extra_mains)__ creates a new row - conditionally. When value is nil, the function will attempt to look up the - value in the object tags. If value is still nil or mtype is nil, the row - is ignored. An mtype of `always` will then always write out the row, - a mtype of `named` only, when the object has a full name. When mtype - is `named_with_key`, the function checks for a domain name, i.e. a name - tag prefixed with the name of the main key. Only if at least one is found, - the row will be written. The names are replaced with the domain names found. - -#### Match functions - -The Place functions usually expect either a _match function_ or a -_group match function_ to find the tags to apply their function to. - -The __match function__ is a Lua function which takes two parameters, -key and value, and returns a boolean to indicate that a tag matches. The -flex-base module has a convenience function `tag_match()` to create such a -function. It takes a table with two optional fields: `keys` takes a key match -list (see above), `tags` takes a table with keys that point to a list of -possible values, thus defining key/value matches. - -The __group match function__ is a Lua function which also takes two parameters, -key and value, and returns a string indicating to which group or type they -belong to. The `tag_group()` can be used to create such a function. It expects -a table where the group names are the keys and the values are a key match list. - - - -### Using the gazetteer output of osm2pgsql +## osm2pgsql gazetteer output Nominatim still allows you to configure the gazetteer output to remain backwards compatible with older imports. It will be automatically used @@ -435,7 +566,7 @@ of Nominatim. Do not use the gazetteer output for new imports. There is no guarantee that new versions of Nominatim are fully compatible with the gazetteer output. -### Changing the Style of Existing Databases +## Changing the style of existing databases There is normally no issue changing the style of a database that is already imported and now kept up-to-date with change files. Just be aware that any diff --git a/docs/customize/Special-Phrases.md b/docs/customize/Special-Phrases.md index 4824512bf..3ab837f22 100644 --- a/docs/customize/Special-Phrases.md +++ b/docs/customize/Special-Phrases.md @@ -17,7 +17,7 @@ columns: * **phrase**: the keyword to look for * **class**: key of the main tag of the place to find - (see [principal tags in import style](Import-Styles.md#set_main_tags-principal-tags) + (see [Import styles](Import-Styles.md#how-processing-works) * **type**: value of the main tag * **operator**: type of special phrase, may be one of: * *in*: place is within the place defined by the search term (e.g. "_Hotels in_ Berlin") diff --git a/docs/extra.css b/docs/extra.css index 1decc4789..033e99035 100644 --- a/docs/extra.css +++ b/docs/extra.css @@ -2,8 +2,8 @@ display: none!important } -.wy-nav-content { - max-width: 900px!important +.md-content { + max-width: 800px } table { diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index 019328125..fbc9aab1b 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -270,7 +270,9 @@ module.IGNORE_KEYS.metatags = {'note', 'note:*', 'source', 'source:*', '*source' 'is_in:postcode'} module.IGNORE_KEYS.name = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', 'name:etymology', 'name:signed', 'name:botanical'} -module.IGNORE_KEYS.address = {'addr:street:*', 'addr:TW:dataset'} +module.IGNORE_KEYS.address = {'addr:street:*', 'addr:city:*', 'addr:district:*', + 'addr:province:*', 'addr:subdistrict:*', 'addr:place:*', + 'addr:TW:dataset'} -- Extra tags (prefiltered away) From eeb3d5dd0affdd3317ff62177905b5474ad2fd34 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 9 Dec 2024 14:05:15 +0100 Subject: [PATCH 10/17] make nominatim callable with themepark style --- src/nominatim_db/tools/exec_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/nominatim_db/tools/exec_utils.py b/src/nominatim_db/tools/exec_utils.py index fc3a7465f..7629e2a2b 100644 --- a/src/nominatim_db/tools/exec_utils.py +++ b/src/nominatim_db/tools/exec_utils.py @@ -39,7 +39,10 @@ def run_osm2pgsql(options: Mapping[str, Any]) -> None: if str(options['osm2pgsql_style']).endswith('.lua'): env['LUA_PATH'] = ';'.join((str(options['osm2pgsql_style_path'] / '?.lua'), - os.environ.get('LUAPATH', ';'))) + os.environ.get('LUA_PATH', ';'))) + env['THEMEPARK_PATH'] = str(options['osm2pgsql_style_path'] / 'themes') + if 'THEMEPARK_PATH' in os.environ: + env['THEMEPARK_PATH'] += ':' + os.environ['THEMEPARK_PATH'] cmd.extend(('--output', 'flex')) for flavour in ('data', 'index'): From e2a9b5fdf736a7e2605910db302c13fb26dfc077 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 9 Dec 2024 16:34:18 +0100 Subject: [PATCH 11/17] exclude sidewalks and similar footways These footways are part of a street that is usually already named. --- lib-lua/themes/nominatim/presets.lua | 16 +++++++++++++++- test/bdd/osm2pgsql/import/tags.feature | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index fbc9aab1b..d8aa75340 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -2,6 +2,20 @@ local module = {} +-- Customized main tag filter functions + +local EXCLUDED_FOOTWAYS = { sidewalk = 1, crossing = 1, link = 1, traffic_aisle } + +local function filter_footways(place) + if place.has_name then + local footway = place.object.tags.footway + if footway == nil or EXCLUDED_FOOTWAYS[footway] ~= 1 then + return place + end + end + return false +end + -- Main tag definition module.MAIN_TAGS = {} @@ -156,7 +170,7 @@ module.MAIN_TAGS_STREETS.default = { service = 'named', cycleway = 'named', path = 'named', - footway = 'named', + footway = filter_footways, steps = 'named', bridleway = 'named', track = 'named', diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index cf530eb7d..948ffe841 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -206,3 +206,20 @@ Feature: Tag evaluation | object | class | type | address | | N13001 | place | houses | 'interpolation': 'odd' | | N13002 | place | houses | 'interpolation': 'even' | + + + Scenario: Footways + When loading osm data + """ + n1 x0.0 y0.0 + n2 x0 y0.0001 + w1 Thighway=footway Nn1,n2 + w2 Thighway=footway,name=Road Nn1,n2 + w3 Thighway=footway,name=Road,footway=sidewalk Nn1,n2 + w4 Thighway=footway,name=Road,footway=crossing Nn1,n2 + w5 Thighway=footway,name=Road,footway=residential Nn1,n2 + """ + Then place contains exactly + | object | name+name | + | W2 | Road | + | W5 | Road | From b1e5265d33a2fe1b56034218e1832d35ea4cbdac Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 9 Dec 2024 17:12:35 +0100 Subject: [PATCH 12/17] switch to subtags for tourism=information and natural=water --- lib-lua/themes/nominatim/presets.lua | 57 ++++++++++++++++++++++++-- test/bdd/osm2pgsql/import/tags.feature | 33 +++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index d8aa75340..f230433e8 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -16,6 +16,43 @@ local function filter_footways(place) return false end +local function include_when_tag_present(key, value, named) + if named then + return function(place) + if place.has_name and place.intags[key] == value then + return place + end + return false + end + else + return function(place) + if place.intags[key] == value then + return place + end + return false + end + end +end + +local function exclude_when_key_present(key, named) + if named then + return function(place) + if place.has_name and place.intags[key] == nil then + return place + end + return false + end + else + return function(place) + if place.intags[key] == nil then + return place + end + return false + end + + end +end + -- Main tag definition module.MAIN_TAGS = {} @@ -65,9 +102,21 @@ module.MAIN_TAGS.natural = { yes = 'delete', no = 'delete', coastline = 'delete', - saddle = 'fallback'}, + saddle = 'fallback', + water = exclude_when_key_present('water', true)}, mountain_pass = {'always', - no = 'delete'} + no = 'delete'}, + water = {include_when_tag_present('natural', 'water', true), + river = 'never', + stream = 'never', + canal = 'never', + ditch = 'never', + drain = 'never', + fish_pass = 'never', + yes = 'delete', + intermittent = 'delete', + tidal = 'delete' + } } module.MAIN_TAGS_POIS = function (group) @@ -114,6 +163,7 @@ module.MAIN_TAGS_POIS = function (group) historic = {'always', yes = group, no = group}, + information = {include_when_tag_present('tourism', 'information')}, junction = {'fallback', no = group}, leisure = {'always', @@ -148,7 +198,8 @@ module.MAIN_TAGS_POIS = function (group) no = group}, tourism = {'always', no = group, - yes = group}, + yes = group, + information = 'fallback'}, tunnel = {'named_with_key', no = group} } end diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index 948ffe841..8df726ca1 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -223,3 +223,36 @@ Feature: Tag evaluation | object | name+name | | W2 | Road | | W5 | Road | + + + Scenario: Tourism information + When loading osm data + """ + n100 Ttourism=information + n101 Ttourism=information,name=Generic + n102 Ttourism=information,information=guidepost + n103 Thighway=information,information=house + """ + Then place contains exactly + | object | type | + | N101:tourism | information | + | N102:information | guidepost | + | N103:highway | information | + + + Scenario: Water feautures + When loading osm data + """ + n20 Tnatural=water + n21 Tnatural=water,name=SomePond + n22 Tnatural=water,water=pond + n23 Tnatural=water,water=pond,name=Pond + n24 Tnatural=water,water=river,name=BigRiver + n25 Tnatural=water,water=yes + n26 Tnatural=water,water=yes,name=Random + """ + Then place contains exactly + | object | type | + | N21:natural | water | + | N23:water | pond | + | N26:natural | water | From 0d500d4bd19728baa82ee42c4e2d80edbc33aae8 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 9 Dec 2024 17:28:30 +0100 Subject: [PATCH 13/17] do not save names when falling back to addresses If an object doesn't have a useable main tag, then the names should always be ignored, independently of the presence of housenumbers. We have to assume that the name belongs to a feature that was intentionally filtered out. --- lib-lua/themes/nominatim/init.lua | 20 ++++++++++++++++++-- settings/address-levels.json | 8 ++++++++ test/bdd/api/search/queries.feature | 2 +- test/bdd/osm2pgsql/import/tags.feature | 13 +++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/lib-lua/themes/nominatim/init.lua b/lib-lua/themes/nominatim/init.lua index 6d3804e27..dacaaae8e 100644 --- a/lib-lua/themes/nominatim/init.lua +++ b/lib-lua/themes/nominatim/init.lua @@ -134,6 +134,22 @@ function PlaceTransform.named_with_key(place, k) end end +-- Special transform used with address fallbacks: ignore all names +-- except for those marked as being part of the address. +local function address_fallback(place) + if next(place.names) == nil or NAMES.house == nil then + return place + end + + local names = {} + for k, v in pairs(place.names) do + if NAME_FILTER(k, v) == 'house' then + names[k] = v + end + end + return place:clone{names=names} +end + --------- Built-in extratags transformation functions --------------- local function default_extratags_filter(p, k) @@ -379,7 +395,7 @@ function Place:grab_name_parts(data) self.has_name = true elseif atype == 'house' then self.has_name = true - fallback = {'place', 'house', PlaceTransform.always} + fallback = {'place', 'house', address_fallback} end end end @@ -636,7 +652,7 @@ function module.process_tags(o) -- address keys if o:grab_address_parts{groups=ADDRESS_FILTER} > 0 and fallback == nil then - fallback = {'place', 'house', PlaceTransform.always} + fallback = {'place', 'house', address_fallback} end if o.address.country ~= nil and #o.address.country ~= 2 then o.address['country'] = nil diff --git a/settings/address-levels.json b/settings/address-levels.json index b63eac4ef..a82133ef0 100644 --- a/settings/address-levels.json +++ b/settings/address-levels.json @@ -74,6 +74,14 @@ "stone" : 30, "" : [22, 0] }, + "water" : { + "lake" : [20, 0], + "reservoir" : [20, 0], + "wastewater" : [24, 0], + "pond" : [24, 0], + "fountain" : [24, 0], + "" : [22, 0] + }, "waterway" : { "river" : [19, 0], "stream" : [22, 0], diff --git a/test/bdd/api/search/queries.feature b/test/bdd/api/search/queries.feature index 6e640acca..3b06af786 100644 --- a/test/bdd/api/search/queries.feature +++ b/test/bdd/api/search/queries.feature @@ -192,7 +192,7 @@ Feature: Search queries Then exactly 1 result is returned And results contain | class | - | natural | + | water | Examples: | data | diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index 8df726ca1..2a7673b75 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -256,3 +256,16 @@ Feature: Tag evaluation | N21:natural | water | | N23:water | pond | | N26:natural | water | + + Scenario: Drop name for address fallback + When loading osm data + """ + n1 Taddr:housenumber=23,name=Foo + n2 Taddr:housenumber=23,addr:housename=Foo + n3 Taddr:housenumber=23 + """ + Then place contains exactly + | object | type | address | name | + | N1:place | house | 'housenumber': '23' | - | + | N2:place | house | 'housenumber': '23' | 'addr:housename': 'Foo' | + | N3:place | house | 'housenumber': '23' | - | From ad214753fcdd790916aa1b3b35e679616f4eeb07 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 9 Dec 2024 17:57:08 +0100 Subject: [PATCH 14/17] include lock names mapped with "lock_name" Fixes #3365. --- lib-lua/themes/nominatim/presets.lua | 12 ++++++++++++ test/bdd/osm2pgsql/import/tags.feature | 15 +++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index f230433e8..4f4fe8630 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -53,6 +53,17 @@ local function exclude_when_key_present(key, named) end end +local function lock_transform(place) + if place.object.tags.waterway ~= nil then + local name = place.object.tags.lock_name + if name ~= nil then + return place:clone{names={name=name, ref=place.object.tags.lock_ref}} + end + end + + return false +end + -- Main tag definition module.MAIN_TAGS = {} @@ -169,6 +180,7 @@ module.MAIN_TAGS_POIS = function (group) leisure = {'always', nature_reserve = 'fallback', no = group}, + lock = {yes = lock_transform}, man_made = {pier = 'always', tower = 'always', bridge = 'always', diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index 2a7673b75..f4ebe7adf 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -269,3 +269,18 @@ Feature: Tag evaluation | N1:place | house | 'housenumber': '23' | - | | N2:place | house | 'housenumber': '23' | 'addr:housename': 'Foo' | | N3:place | house | 'housenumber': '23' | - | + + + Scenario: Waterway locks + When loading osm data + """ + n1 Twaterway=river,lock=yes + n2 Twaterway=river,lock=yes,lock_name=LeLock + n3 Twaterway=river,lock=yes,name=LeWater + n4 Tamenity=parking,lock=yes,lock_name=Gold + """ + Then place contains exactly + | object | type | name | + | N2:lock | yes | 'name': 'LeLock' | + | N3:waterway | river | 'name': 'LeWater' | + | N4:amenity | parking | - | From 99cf552c17a6ec6f9b844745b8a3517f658022bf Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 11 Dec 2024 10:59:32 +0100 Subject: [PATCH 15/17] exclude unnamed swimming pools Publicly accessible ones are usually mapped as the bigger area with the water park. --- lib-lua/themes/nominatim/presets.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index 4f4fe8630..58e7d400f 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -179,6 +179,7 @@ module.MAIN_TAGS_POIS = function (group) no = group}, leisure = {'always', nature_reserve = 'fallback', + swimming_pool = 'named', no = group}, lock = {yes = lock_transform}, man_made = {pier = 'always', From 48333bfbd4d89f539f960a565aeb932e95129fc6 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 12 Dec 2024 10:23:50 +0100 Subject: [PATCH 16/17] reintroduce brand and remove etymology --- lib-lua/themes/nominatim/presets.lua | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index 58e7d400f..7afb204a7 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -2,6 +2,22 @@ local module = {} +-- Helper functions + +local function group_merge(group1, group2) + for name, values in pairs(group2) do + if group1[name] == nil then + group1[name] = values + else + for _, v in pairs(values) do + table.insert(group1[name], v) + end + end + end + + return group1 +end + -- Customized main tag filter functions local EXCLUDED_FOOTWAYS = { sidewalk = 1, crossing = 1, link = 1, traffic_aisle } @@ -310,10 +326,9 @@ module.NAME_TAGS.core = {main = {'name', 'name:*', 'loc_ref', 'old_ref', 'ISO3166-2'} } module.NAME_TAGS.address = {house = {'addr:housename'}} -module.NAME_TAGS.poi = {extra = {'ref', 'int_ref', 'nat_ref', 'reg_ref', - 'loc_ref', 'old_ref', - 'iata', 'icao', - 'ISO3166-2'}} +module.NAME_TAGS.poi = group_merge({main = {'brand'}, + extra = {'iata', 'icao'}}, + module.NAME_TAGS.core) -- Address tagging @@ -347,7 +362,8 @@ module.IGNORE_KEYS.metatags = {'note', 'note:*', 'source', 'source:*', '*source' 'type', 'is_in:postcode'} module.IGNORE_KEYS.name = {'*:prefix', '*:suffix', 'name:prefix:*', 'name:suffix:*', - 'name:etymology', 'name:signed', 'name:botanical'} + 'name:etymology', 'name:etymology:*', + 'name:signed', 'name:botanical'} module.IGNORE_KEYS.address = {'addr:street:*', 'addr:city:*', 'addr:district:*', 'addr:province:*', 'addr:subdistrict:*', 'addr:place:*', 'addr:TW:dataset'} From 2535780282ee3635df6e4857a864301624f1dadd Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 16 Dec 2024 10:44:37 +0100 Subject: [PATCH 17/17] exclude more tourism=information types --- lib-lua/themes/nominatim/presets.lua | 7 +++++-- test/bdd/osm2pgsql/import/tags.feature | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib-lua/themes/nominatim/presets.lua b/lib-lua/themes/nominatim/presets.lua index 7afb204a7..aa51ac14c 100644 --- a/lib-lua/themes/nominatim/presets.lua +++ b/lib-lua/themes/nominatim/presets.lua @@ -190,7 +190,10 @@ module.MAIN_TAGS_POIS = function (group) historic = {'always', yes = group, no = group}, - information = {include_when_tag_present('tourism', 'information')}, + information = {include_when_tag_present('tourism', 'information'), + yes = 'delete', + route_marker = 'never', + trail_blaze = 'never'}, junction = {'fallback', no = group}, leisure = {'always', @@ -228,7 +231,7 @@ module.MAIN_TAGS_POIS = function (group) tourism = {'always', no = group, yes = group, - information = 'fallback'}, + information = exclude_when_key_present('information')}, tunnel = {'named_with_key', no = group} } end diff --git a/test/bdd/osm2pgsql/import/tags.feature b/test/bdd/osm2pgsql/import/tags.feature index f4ebe7adf..69238e797 100644 --- a/test/bdd/osm2pgsql/import/tags.feature +++ b/test/bdd/osm2pgsql/import/tags.feature @@ -232,15 +232,19 @@ Feature: Tag evaluation n101 Ttourism=information,name=Generic n102 Ttourism=information,information=guidepost n103 Thighway=information,information=house + n104 Ttourism=information,information=yes,name=Something + n105 Ttourism=information,information=route_marker,name=3 """ Then place contains exactly | object | type | + | N100:tourism | information | | N101:tourism | information | | N102:information | guidepost | | N103:highway | information | + | N104:tourism | information | - Scenario: Water feautures + Scenario: Water features When loading osm data """ n20 Tnatural=water