diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02cbbdf386..9880c98269 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: run: sudo -i -u postgres createuser -s $USER && createdb -E utf8 gis && psql -Xq -d gis -c "CREATE EXTENSION postgis; CREATE EXTENSION hstore;" - name: Import empty file run: | - osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis -r xml <(echo '') + osm2pgsql -O flex -S openstreetmap-carto-flex.lua -d gis -r xml <(echo '') - name: Create indexes run: psql -1Xq -v ON_ERROR_STOP=1 -d gis -f indexes.sql - name: Load functions diff --git a/Dockerfile.import b/Dockerfile.import index feb74aa5a7..685471657e 100644 --- a/Dockerfile.import +++ b/Dockerfile.import @@ -14,7 +14,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ osm2pgsql gdal-bin python3-psycopg2 python3-yaml unzip \ python3-requests postgresql-client && rm -rf /var/lib/apt/lists/* -ADD openstreetmap-carto.style / +ADD openstreetmap-carto-flex.lua / RUN mkdir -p /openstreetmap-carto WORKDIR /openstreetmap-carto diff --git a/INSTALL.md b/INSTALL.md index 7bbfb282d5..7673bd5855 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -3,9 +3,9 @@ This document describes how to manually configure your system for running OpenStreetMap Carto. If you prefer quick, platform independent setup for a development environment, without the need to install and configure tools by hand, follow a Docker installation guide in [DOCKER.md](DOCKER.md). ## OpenStreetMap data -You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the pgsql backend (table names of `planet_osm_point`, etc), the default database name (`gis`), and the [lua transforms](https://osm2pgsql.org/doc/manual.html#lua-tag-transformations) documented in the instructions below. +You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets expect a database generated with osm2pgsql using the flex backend. -Start by creating a database +Start by creating a database, we are using the database name `gis` here: ```sh sudo -u postgres createuser -s $USER @@ -21,7 +21,7 @@ psql -d gis -c 'CREATE EXTENSION postgis; CREATE EXTENSION hstore;' Then, grab some OSM data; It's probably easiest to grab an PBF of OSM data from [Geofabrik](https://download.geofabrik.de/). Once you've done that, import with osm2pgsql: ```sh -osm2pgsql -G --hstore --style openstreetmap-carto.style --tag-transform-script openstreetmap-carto.lua -d gis ~/path/to/data.osm.pbf +osm2pgsql -O flex -S openstreetmap-carto-flex.lua -d gis ~/path/to/data.osm.pbf ``` You can find a more detailed guide to setting up a database and loading data with osm2pgsql at [switch2osm.org](https://switch2osm.org/serving-tiles/manually-building-a-tile-server-16-04-2-lts/). @@ -89,7 +89,7 @@ To display *any* map, a database containing OpenStreetMap data and some utilitie * [PostgreSQL](https://www.postgresql.org/) * [PostGIS](https://postgis.net/) -* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) to [import your data](https://switch2osm.org/serving-tiles/updating-as-people-edit/) into a PostGIS database +* [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) (>= 1.8.0) to [import your data](https://switch2osm.org/serving-tiles/updating-as-people-edit/) into a PostGIS database * Python 3 with the psycopg2, yaml, and requests libraries (`python3-psycopg2`, `python3-yaml`, `python3-requests` packages on Debian-derived systems) * `ogr2ogr` for loading shapefiles into the database (`gdal-bin` on Debian-derived systems) diff --git a/openstreetmap-carto-flex.lua b/openstreetmap-carto-flex.lua new file mode 100644 index 0000000000..832cbe5a24 --- /dev/null +++ b/openstreetmap-carto-flex.lua @@ -0,0 +1,689 @@ + +-- This is the osm2pgsql configuration for the OpenStreetMap Carto map style +-- using the osm2pgsql flex output. + +-- It is written in a way that it can be used with or without the Themepark +-- framework. For more about Themepark see https://osm2pgsql.org/themepark/ . + +-- --------------------------------------------------------------------------- + +-- CONFIGURATION + +-- Prefix for all output table names. +-- +-- (This used to be set with the --prefix command line option, but note the +-- trailing '_' letter which was not needed with the command line option.) +local PREFIX = 'planet_osm_' + +-- Set this to the database schema. +-- +-- (This used to be set with the --output-pgsql-schema command line option.) +local SCHEMA = 'public' + +-- --------------------------------------------------------------------------- + +-- Needed for use with the Themepark framework +local themepark = ... + +-- --------------------------------------------------------------------------- + +-- A list of columns per table in the order they will appear in the database +-- tables. Columns can either be +-- * a string ('highway') in which case they will be added as 'text' column or +-- * a Lua table with a column definition for the define_table() command. +local table_columns = { + point = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'barrier', + 'boundary', + 'building', + 'highway', + 'historic', + 'junction', + 'landuse', + { column = 'layer', type = 'int4' }, + 'leisure', + 'lock', + 'man_made', + 'military', + 'name', + 'natural', + 'oneway', + 'place', + 'power', + 'railway', + 'ref', + 'religion', + 'shop', + 'tourism', + 'water', + 'waterway', + { column = 'tags', type = 'hstore' }, + }, + line = { + 'access', + 'addr:housename', + 'addr:housenumber', + 'addr:interpolation', + 'admin_level', + 'aerialway', + 'aeroway', + 'amenity', + 'barrier', + 'bicycle', + 'bridge', + 'boundary', + 'building', + 'construction', + 'covered', + 'foot', + 'highway', + 'historic', + 'horse', + 'junction', + 'landuse', + { column = 'layer', type = 'int4' }, + 'leisure', + 'lock', + 'man_made', + 'military', + 'name', + 'natural', + 'oneway', + 'place', + 'power', + 'railway', + 'ref', + 'religion', + 'route', + 'service', + 'shop', + 'surface', + 'tourism', + 'tracktype', + 'tunnel', + 'water', + 'waterway', + { column = 'way_area', type = 'real' }, + { column = 'z_order', type = 'int4' }, + { column = 'tags', type = 'hstore' }, + }, +} + +-- The columns for the roads and polygon tables are the same as for the line +-- table, so just reuse them. (Note: This is not a deep copy!) +table_columns.roads = table_columns.line +table_columns.polygon = table_columns.line + +-- These are the database table definitions. They will be combined with the +-- column definitions above to create the final definitions. +-- +-- (The index definitions reflect the index definitions in indexes.[sql|yml] +-- but have been commented out so that the behaviour of the configuration is +-- the same as before. The index definitions here can be used instead of the +-- ones in indexes.[sql|yml], the indexes will be the same except that it is +-- currently not possible to name the indexes.) +local table_definitions = { + point = { + geometry_type = 'point', + ids = { type = 'node' }, + }, + line = { + geometry_type = 'linestring', + ids = { type = 'way' }, + }, + roads = { + geometry_type = 'linestring', + ids = { type = 'way' }, + }, + polygon = { + geometry_type = 'geometry', + ids = { type = 'area' }, + }, +} + +-- This will contain the database tables after they have been initialized. +local tables = {} + +-- Contain a hash with all text columns for the point table and all other +-- tables, respectively. +-- Used to quickly check whether a columns of a given name exists. +local columns_in_point_table = {} +local columns_in_non_point_tables = {} + +-- Combine the table definitions and the column definitions from above to +-- the final definitions and create the tables. +for name, definition in pairs(table_definitions) do + definition.name = PREFIX .. name + definition.schema = SCHEMA + definition.ids.id_column = 'osm_id' + definition.columns = {} + definition.geom = { + column = 'way', + type = definition.geometry_type, + not_null = true + } + + -- Add column definitions to table definitions + for _, column in ipairs(table_columns[name]) do + if type(column) == 'table' then + table.insert(definition.columns, column) + else + table.insert(definition.columns, { column = column, type = 'text' }) + + if name == 'point' then + columns_in_point_table[column] = true + elseif name == 'line' then + columns_in_non_point_tables[column] = true + end + end + end + + if themepark then + themepark:add_table(definition) + else + table.insert(definition.columns, definition.geom) + tables[name] = osm2pgsql.define_table(definition) + end +end + +-- Objects with any of the following keys will be treated as polygon +local polygon_keys = { + 'abandoned:aeroway', + 'abandoned:amenity', + 'abandoned:building', + 'abandoned:landuse', + 'abandoned:power', + 'aeroway', + 'allotments', + 'amenity', + 'area:highway', + 'craft', + 'building', + 'building:part', + 'club', + 'golf', + 'emergency', + 'harbour', + 'healthcare', + 'historic', + 'landuse', + 'leisure', + 'man_made', + 'military', + 'natural', + 'office', + 'place', + 'power', + 'public_transport', + 'shop', + 'tourism', + 'water', + 'waterway', + 'wetland', +} + +-- Objects with any of the following key/value combinations will be treated as linestring +local linestring_values = { + golf = { cartpath = true, hole = true, path = true }, + emergency = { designated = true, destination = true, no = true, + official = true, yes = true }, + historic = { citywalls = true }, + leisure = { track = true, slipway = true }, + man_made = { breakwater = true, cutline = true, embankment = true, + groyne = true, pipeline = true }, + natural = { cliff = true, earth_bank = true, tree_row = true, + ridge = true, arete = true }, + power = { cable = true, line = true, minor_line = true }, + tourism = { yes = true }, + waterway = { canal = true, derelict_canal = true, ditch = true, + drain = true, river = true, stream = true, + tidal_channel = true, wadi = true, weir = true }, +} + +-- Objects with any of the following key/value combinations will be treated as polygon +local polygon_values = { + aerialway = { station = true }, + boundary = { aboriginal_lands = true, national_park = true, + protected_area = true }, + highway = { services = true, rest_area = true }, + junction = { yes = true }, + railway = { station = true }, +} + +-- Tags with the following keys will be igored +local ignore_keys = { + 'note', + 'source', + 'source:addr', + 'source:date', + 'source_ref', + 'attribution', + 'comment', + 'fixme', + + -- Tags generally dropped by editors, not otherwise covered + 'created_by', + 'odbl', + + -- Lots of import tags + -- EUROSHA (Various countries) + 'project:eurosha_2012', + -- UrbIS (Brussels, BE) + 'ref:UrbIS', + -- NHN (CA) + 'accuracy:meters', + 'waterway:type', + -- StatsCan (CA) + 'statscan:rbuid', + -- RUIAN (CZ) + 'ref:ruian:addr', + 'ref:ruian', + 'building:ruian:type', + -- DIBAVOD (CZ) + 'dibavod:id', + -- UIR-ADR (CZ) + 'uir_adr:ADRESA_KOD', + -- GST (DK) + 'gst:feat_id', + -- osak (DK) + 'osak:identifier', + -- Maa-amet (EE) + 'maaamet:ETAK', + -- FANTOIR (FR) + 'ref:FR:FANTOIR', + -- OPPDATERIN (NO) + 'OPPDATERIN', + -- Various imports (PL) + 'addr:city:simc', + 'addr:street:sym_ul', + 'building:usage:pl', + 'building:use:pl', + -- TERYT (PL) + 'teryt:simc', + -- RABA (SK) + 'raba:id', + -- LINZ (NZ) + 'linz2osm:objectid', + -- DCGIS (Washington DC, US) + 'dcgis:gis_id', + -- Building Identification Number (New York, US) + 'nycdoitt:bin', + -- Chicago Building Import (US) + 'chicago:building_id', + -- Louisville, Kentucky/Building Outlines Import (US) + 'lojic:bgnum', + -- MassGIS (Massachusetts, US) + 'massgis:way_id', + -- TIGER (US) + 'tiger:cfcc', + 'tiger:county', + 'tiger:reviewed', + + -- misc + 'import', + 'import_uuid', + 'OBJTYPE', + 'SK53_bulk:load', +} + +-- Tags with the following key prefixes will be ignored. +local ignore_key_prefixes = { + 'note:', + 'source:', + + -- Corine (CLC) (Europe) + 'CLC:', + -- Geobase (CA) + 'geobase:', + -- CanVec (CA) + 'canvec:', + -- Geobase (CA) + 'geobase:', + -- kms (DK) + 'kms:', + -- ngbe (ES) + -- See also note:es and source:file above + 'ngbe:', + -- Friuli Venezia Giulia (IT) + 'it:fvg:', + -- KSJ2 (JA) + -- See also note:ja and source_ref above + 'KSJ2:', + -- Yahoo/ALPS (JA) + 'yh:', + -- LINZ (NZ) + 'LINZ2OSM:', + 'LINZ:', + -- WroclawGIS (PL) + 'WroclawGIS:', + -- Naptan (UK) + 'naptan:', + -- TIGER (US) + 'tiger:', + -- GNIS (US) + 'gnis:', + -- National Hydrography Dataset (US) + 'NHD:', + 'nhd:', + -- mvdgis (Montevideo, UY) + 'mvdgis:', +} + +-- Big table for z_order and roads status for certain tags. +-- The road status (true/false) determines whether or not the feature will be +-- added to the 'roads' table. +-- z=0 is turned into nil by the z_order function. +-- Road z values are divided by 10 for objects tagged as highway=construction, +-- construction=[HIGHWAY_CLASS], so must be multiples of 10. +local roads_info = { + highway = { + motorway = { z = 380, roads = true }, + trunk = { z = 370, roads = true }, + primary = { z = 360, roads = true }, + secondary = { z = 350, roads = true }, + tertiary = { z = 340, roads = false }, + residential = { z = 330, roads = false }, + unclassified = { z = 330, roads = false }, + road = { z = 330, roads = false }, + living_street = { z = 320, roads = false }, + pedestrian = { z = 310, roads = false }, + raceway = { z = 300, roads = false }, + motorway_link = { z = 240, roads = true }, + trunk_link = { z = 230, roads = true }, + primary_link = { z = 220, roads = true }, + secondary_link = { z = 210, roads = true }, + tertiary_link = { z = 200, roads = false }, + service = { z = 150, roads = false }, + track = { z = 110, roads = false }, + path = { z = 100, roads = false }, + footway = { z = 100, roads = false }, + bridleway = { z = 100, roads = false }, + cycleway = { z = 100, roads = false }, + steps = { z = 90, roads = false }, + platform = { z = 90, roads = false }, + }, + railway = { + rail = { z = 440, roads = true }, + subway = { z = 420, roads = true }, + narrow_gauge = { z = 420, roads = true }, + light_rail = { z = 420, roads = true }, + funicular = { z = 420, roads = true }, + preserved = { z = 420, roads = false }, + monorail = { z = 420, roads = false }, + miniature = { z = 420, roads = false }, + turntable = { z = 420, roads = false }, + tram = { z = 410, roads = false }, + disused = { z = 400, roads = false }, + construction = { z = 400, roads = false }, + platform = { z = 90, roads = false }, + }, + aeroway = { + runway = { z = 60, roads = false }, + taxiway = { z = 50, roads = false }, + }, + boundary = { + administrative = { z = 0, roads = true }, + }, +} + +local excluded_railway_service = { + spur = true, + siding = true, + yard = true, +} + +-- Bring the polygon keys into hash table +local polygon_lookup = {} +for n = 1, #polygon_keys do + polygon_lookup[polygon_keys[n]] = true +end + +-- Bring the keys we want to ignore into hash table for fast lookup +-- The 'layer' tag is is a special case +local ignore_keys_lookup = { layer = true } +for n = 1, #ignore_keys do + ignore_keys_lookup[ignore_keys[n]] = true +end + +local ignore_key_prefixes_lookup = {} +for _, prefix in ipairs(ignore_key_prefixes) do + local length = string.len(prefix) + if not ignore_key_prefixes_lookup[length] then + ignore_key_prefixes_lookup[length] = {} + end + ignore_key_prefixes_lookup[length][prefix] = true +end + +-- --------------------------------------------------------------------------- + +-- Gets the z_order and roads table status for a set of tags. +-- +-- @param tags OSM tags +-- @return z_order: if an object with z_order, otherwise nil +-- in_roads: should object be added to roads_table? (true or false) +-- +local function calculate_z_order(tags) + local z_order = 0 + local in_roads = false + + for key, value in pairs(tags) do + local ri = roads_info[key] + if ri and ri[value] then + z_order = math.max(z_order, ri[value].z) + if in_roads == false and ri[value].roads then + if not (key ~= 'railway' or tags.service) then + in_roads = true + end + if not excluded_railway_service[tags.service] then + in_roads = true + end + end + end + end + + if tags.highway == 'construction' then + if tags.construction and roads_info.highway[tags.construction] then + z_order = math.max(z_order, roads_info.highway[tags.construction].z / 10) + else + -- For unknown roads, assume highway=road + z_order = math.max(z_order, roads_info.highway.road.z / 10) + end + end + + return z_order ~= 0 and z_order or nil, in_roads +end + +-- Check if an object with given tags should be treated as polygon +-- +-- @param tags OSM tags +-- @return true if area, false if linear +-- +local function is_area(tags) + local area_tag = tags.area + if area_tag then + return area_tag == 'yes' + end + + for key, value in pairs(tags) do + if value ~= 'no' then + if polygon_lookup[key] then + local lv = linestring_values[key] + if not (lv and lv[value]) then + return true + end + end + + local pv = polygon_values[key] + if pv and pv[value] then + return true + end + end + end + + return false +end + +-- Normalizes layer tags to integers +-- +-- @param value The layer tag value +-- @return The input value if it is an integer between -100 and 100, or nil +-- otherwise. (Can be changed to return 0 if that's more convenient.) +-- +local function normalize_layer(value) + -- check if value exists, is numeric, and is in range + if value and string.find(value, '^-?%d+$') then + value = tonumber(value) + if value < 100 and value > -100 then + return value + end + end + return nil +end + +-- Decide whether to keep this tag. +-- +-- @param key The tag key +-- @return true of false +local function keep_tag(key) + if ignore_keys_lookup[key] then + return false + end + + for length, lookup in pairs(ignore_key_prefixes_lookup) do + local prefix = string.sub(key, 1, length) + if lookup[prefix] then + return false + end + end + + return true +end + +-- Prepare columns based on tags. Some tags go into their own columns, the +-- rest will be put into the hstore column called "tags". +-- +-- @param tags OSM tags +-- @param tag_map Lua table that contains the OSM tags that will get a dedicated column +-- @param ignore_type Set to 'true' to ignore 'type' tag +-- @return the contents for the columns +-- +local function prepare_columns(tags, tag_map, ignore_type) + local attrs = { tags = {}, layer = normalize_layer(tags.layer) } + local found_tag = false + + for key, value in pairs(tags) do + if tag_map[key] then + attrs[key] = value + found_tag = true + elseif ignore_type and key == 'type' then -- luacheck: ignore 542 + -- do nothing + elseif keep_tag(key) then + attrs.tags[key] = value + found_tag = true + end + end + + if not found_tag then + return nil + end + + return attrs +end + +local insert_row +if themepark then + insert_row = function(table_name, columns) + themepark:insert(PREFIX .. table_name, columns, {}, {}) + end +else + insert_row = function(table_name, columns) + tables[table_name]:insert(columns) + end +end + +-- Add an object to the 'line' or 'roads' table. +local function add_linear(table_name, attrs, geom) + for sgeom in geom:geometries() do + attrs.way = sgeom + insert_row(table_name, attrs) + end +end + +-- Add an object to the 'polygon' table. +local function add_polygon(attrs, geom) + attrs.way = geom + attrs.way_area = geom:area() + insert_row('polygon', attrs) +end + +local function process_node(object) + local attrs = prepare_columns(object.tags, columns_in_point_table, false) + if attrs == nil then + return + end + + attrs.way = object:as_point() + insert_row('point', attrs) +end + +local function process_way(object) + local attrs = prepare_columns(object.tags, columns_in_non_point_tables, false) + if attrs == nil then + return + end + + local in_roads + attrs.z_order, in_roads = calculate_z_order(object.tags) + + if object.is_closed and is_area(object.tags) then + add_polygon(attrs, object:as_polygon():transform(3857)) + else + local geom = object:as_linestring():transform(3857):segmentize(100000) + add_linear('line', attrs, geom) + + if in_roads then + add_linear('roads', attrs, geom) + end + end +end + +local function process_relation(object) + local attrs = prepare_columns(object.tags, columns_in_non_point_tables, true) + if attrs == nil then + return + end + + local in_roads + attrs.z_order, in_roads = calculate_z_order(object.tags) + + local type = object.tags.type + if type == 'boundary' or (type == 'multipolygon' and object.tags.boundary) or type == 'route' then + local geom = object:as_multilinestring():line_merge():transform(3857):segmentize(100000) + add_linear('line', attrs, geom) + + if in_roads then + add_linear('roads', attrs, geom) + end + + add_polygon(attrs, object:as_multipolygon():transform(3857)) + elseif type == 'multipolygon' then + add_polygon(attrs, object:as_multipolygon():transform(3857)) + end +end + +if themepark then + themepark:add_proc('node', process_node) + themepark:add_proc('way', process_way) + themepark:add_proc('relation', process_relation) +else + osm2pgsql.process_node = process_node + osm2pgsql.process_way = process_way + osm2pgsql.process_relation = process_relation +end diff --git a/openstreetmap-carto.lua b/openstreetmap-carto.lua deleted file mode 100644 index 8fc2c85212..0000000000 --- a/openstreetmap-carto.lua +++ /dev/null @@ -1,436 +0,0 @@ --- For documentation of Lua tag transformations, see: --- https://github.com/openstreetmap/osm2pgsql/blob/master/docs/lua.md - --- Objects with any of the following keys will be treated as polygon -local polygon_keys = { - 'abandoned:aeroway', - 'abandoned:amenity', - 'abandoned:building', - 'abandoned:landuse', - 'abandoned:power', - 'aeroway', - 'allotments', - 'amenity', - 'area:highway', - 'craft', - 'building', - 'building:part', - 'club', - 'golf', - 'emergency', - 'harbour', - 'healthcare', - 'historic', - 'landuse', - 'leisure', - 'man_made', - 'military', - 'natural', - 'office', - 'place', - 'power', - 'public_transport', - 'shop', - 'tourism', - 'water', - 'waterway', - 'wetland' -} - --- Objects with any of the following key/value combinations will be treated as linestring -local linestring_values = { - golf = {cartpath = true, hole = true, path = true}, - emergency = {designated = true, destination = true, no = true, official = true, yes = true}, - historic = {citywalls = true}, - leisure = {track = true, slipway = true}, - man_made = {breakwater = true, cutline = true, embankment = true, groyne = true, pipeline = true}, - natural = {cliff = true, earth_bank = true, tree_row = true, ridge = true, arete = true}, - power = {cable = true, line = true, minor_line = true}, - tourism = {yes = true}, - waterway = {canal = true, derelict_canal = true, ditch = true, drain = true, river = true, stream = true, tidal_channel = true, wadi = true, weir = true} -} - --- Objects with any of the following key/value combinations will be treated as polygon -local polygon_values = { - aerialway = {station = true}, - boundary = {aboriginal_lands = true, national_park = true, protected_area= true}, - highway = {services = true, rest_area = true}, - junction = {yes = true}, - railway = {station = true} -} - --- The following keys will be deleted -local delete_tags = { - 'note', - 'source', - 'source_ref', - 'attribution', - 'comment', - 'fixme', - -- Tags generally dropped by editors, not otherwise covered - 'created_by', - 'odbl', - -- Lots of import tags - -- EUROSHA (Various countries) - 'project:eurosha_2012', - - -- UrbIS (Brussels, BE) - 'ref:UrbIS', - - -- NHN (CA) - 'accuracy:meters', - 'waterway:type', - -- StatsCan (CA) - 'statscan:rbuid', - - -- RUIAN (CZ) - 'ref:ruian:addr', - 'ref:ruian', - 'building:ruian:type', - -- DIBAVOD (CZ) - 'dibavod:id', - -- UIR-ADR (CZ) - 'uir_adr:ADRESA_KOD', - - -- GST (DK) - 'gst:feat_id', - -- osak (DK) - 'osak:identifier', - - -- Maa-amet (EE) - 'maaamet:ETAK', - -- FANTOIR (FR) - 'ref:FR:FANTOIR', - - -- OPPDATERIN (NO) - 'OPPDATERIN', - -- Various imports (PL) - 'addr:city:simc', - 'addr:street:sym_ul', - 'building:usage:pl', - 'building:use:pl', - -- TERYT (PL) - 'teryt:simc', - - -- RABA (SK) - 'raba:id', - - -- LINZ (NZ) - 'linz2osm:objectid', - -- DCGIS (Washington DC, US) - 'dcgis:gis_id', - -- Building Identification Number (New York, US) - 'nycdoitt:bin', - -- Chicago Building Import (US) - 'chicago:building_id', - -- Louisville, Kentucky/Building Outlines Import (US) - 'lojic:bgnum', - -- MassGIS (Massachusetts, US) - 'massgis:way_id', - - -- misc - 'import', - 'import_uuid', - 'OBJTYPE', - 'SK53_bulk:load' -} -delete_prefixes = { - 'note:', - 'source:', - -- Corine (CLC) (Europe) - 'CLC:', - - -- Geobase (CA) - 'geobase:', - -- CanVec (CA) - 'canvec:', - -- Geobase (CA) - 'geobase:', - - -- kms (DK) - 'kms:', - - -- ngbe (ES) - -- See also note:es and source:file above - 'ngbe:', - - -- Friuli Venezia Giulia (IT) - 'it:fvg:', - - -- KSJ2 (JA) - -- See also note:ja and source_ref above - 'KSJ2:', - -- Yahoo/ALPS (JA) - 'yh:', - - -- LINZ (NZ) - 'LINZ2OSM:', - 'LINZ:', - - -- WroclawGIS (PL) - 'WroclawGIS:', - -- Naptan (UK) - 'naptan:', - - -- TIGER (US) - 'tiger:', - -- GNIS (US) - 'gnis:', - -- National Hydrography Dataset (US) - 'NHD:', - 'nhd:', - -- mvdgis (Montevideo, UY) - 'mvdgis:' -} - --- Big table for z_order and roads status for certain tags. z=0 is turned into --- nil by the z_order function -local roads_info = { - highway = { - motorway = {z = 380, roads = true}, - trunk = {z = 370, roads = true}, - primary = {z = 360, roads = true}, - secondary = {z = 350, roads = true}, - tertiary = {z = 340, roads = false}, - residential = {z = 330, roads = false}, - unclassified = {z = 330, roads = false}, - road = {z = 330, roads = false}, - living_street = {z = 320, roads = false}, - pedestrian = {z = 310, roads = false}, - raceway = {z = 300, roads = false}, - motorway_link = {z = 240, roads = true}, - trunk_link = {z = 230, roads = true}, - primary_link = {z = 220, roads = true}, - secondary_link = {z = 210, roads = true}, - tertiary_link = {z = 200, roads = false}, - service = {z = 150, roads = false}, - track = {z = 110, roads = false}, - path = {z = 100, roads = false}, - footway = {z = 100, roads = false}, - bridleway = {z = 100, roads = false}, - cycleway = {z = 100, roads = false}, - steps = {z = 90, roads = false}, - platform = {z = 90, roads = false} - }, - railway = { - rail = {z = 440, roads = true}, - subway = {z = 420, roads = true}, - narrow_gauge = {z = 420, roads = true}, - light_rail = {z = 420, roads = true}, - funicular = {z = 420, roads = true}, - preserved = {z = 420, roads = false}, - monorail = {z = 420, roads = false}, - miniature = {z = 420, roads = false}, - turntable = {z = 420, roads = false}, - tram = {z = 410, roads = false}, - disused = {z = 400, roads = false}, - construction = {z = 400, roads = false}, - platform = {z = 90, roads = false}, - }, - aeroway = { - runway = {z = 60, roads = false}, - taxiway = {z = 50, roads = false}, - }, - boundary = { - administrative = {z = 0, roads = true} - }, -} - -local excluded_railway_service = { - spur = true, - siding = true, - yard = true -} ---- Gets the z_order for a set of tags --- @param tags OSM tags --- @return z_order if an object with z_order, otherwise nil -function z_order(tags) - local z = 0 - for k, v in pairs(tags) do - if roads_info[k] and roads_info[k][v] then - z = math.max(z, roads_info[k][v].z) - end - end - - if tags["highway"] == "construction" then - if tags["construction"] and roads_info["highway"][tags["construction"]] then - z = math.max(z, roads_info["highway"][tags["construction"]].z/10) - else - z = math.max(z, 33) - end - end - - return z ~= 0 and z or nil -end - ---- Gets the roads table status for a set of tags --- @param tags OSM tags --- @return 1 if it belongs in the roads table, 0 otherwise -function roads(tags) - for k, v in pairs(tags) do - if roads_info[k] and roads_info[k][v] and roads_info[k][v].roads then - if not (k ~= 'railway' or tags.service) then - return 1 - elseif not excluded_railway_service[tags.service] then - return 1 - end - end - end - return 0 -end - ---- Generic filtering of OSM tags --- @param tags Raw OSM tags --- @return Filtered OSM tags -function filter_tags_generic(tags) - -- Short-circuit for untagged objects - if next(tags) == nil then - return 1, {} - end - - -- Delete tags listed in delete_tags - for _, d in ipairs(delete_tags) do - tags[d] = nil - end - - -- By using a second loop for wildcards we avoid checking already deleted tags - for tag, _ in pairs (tags) do - for _, d in ipairs(delete_prefixes) do - if string.sub(tag, 1, string.len(d)) == d then - tags[tag] = nil - break - end - end - end - - -- Filter out objects that have no tags after deleting - if next(tags) == nil then - return 1, {} - end - - -- Convert layer to an integer - tags['layer'] = layer(tags['layer']) - return 0, tags -end - --- Filtering on nodes -function filter_tags_node (keyvalues, numberofkeys) - return filter_tags_generic(keyvalues) -end - --- Filtering on relations -function filter_basic_tags_rel (keyvalues, numberofkeys) - -- Filter out objects that are filtered out by filter_tags_generic - local filter, keyvalues = filter_tags_generic(keyvalues) - if filter == 1 then - return 1, keyvalues - end - - -- Filter out all relations except route, multipolygon and boundary relations - if ((keyvalues["type"] ~= "route") and (keyvalues["type"] ~= "multipolygon") and (keyvalues["type"] ~= "boundary")) then - return 1, keyvalues - end - - return 0, keyvalues -end - --- Filtering on ways -function filter_tags_way (keyvalues, numberofkeys) - local filter = 0 -- Will object be filtered out? - local polygon = 0 -- Will object be treated as polygon? - - -- Filter out objects that are filtered out by filter_tags_generic - filter, keyvalues = filter_tags_generic(keyvalues) - if filter == 1 then - return filter, keyvalues, polygon, roads - end - - polygon = isarea(keyvalues) - - -- Add z_order column - keyvalues["z_order"] = z_order(keyvalues) - - return filter, keyvalues, polygon, roads(keyvalues) -end - ---- Handling for relation members and multipolygon generation --- @param keyvalues OSM tags, after processing by relation transform --- @param keyvaluemembers OSM tags of relation members, after processing by way transform --- @param roles OSM roles of relation members --- @param membercount number of members --- @return filter, cols, member_superseded, boundary, polygon, roads -function filter_tags_relation_member (keyvalues, keyvaluemembers, roles, membercount) - local members_superseded = {} - - -- Start by assuming that this not an old-style MP - for i = 1, membercount do - members_superseded[i] = 0 - end - - local type = keyvalues["type"] - - -- Remove type key - keyvalues["type"] = nil - - -- Filter out relations with just a type tag or no tags - if next(keyvalues) == nil then - return 1, keyvalues, members_superseded, 0, 0, 0 - end - - if type == "boundary" or (type == "multipolygon" and keyvalues["boundary"]) then - keyvalues.z_order = z_order(keyvalues) - return 0, keyvalues, members_superseded, 1, 0, roads(keyvalues) - -- For multipolygons... - elseif (type == "multipolygon") then - -- Multipolygons by definition are polygons, so we know roads = linestring = 0, polygon = 1 - keyvalues.z_order = z_order(keyvalues) - return 0, keyvalues, members_superseded, 0, 1, 0 - elseif type == "route" then - keyvalues.z_order = z_order(keyvalues) - return 0, keyvalues, members_superseded, 1, 0, roads(keyvalues) - end - - -- Unknown type of relation or no type tag - return 1, keyvalues, members_superseded, 0, 0, 0 -end - ---- Check if an object with given tags should be treated as polygon --- @param tags OSM tags --- @return 1 if area, 0 if linear -function isarea (tags) - -- Treat objects tagged as area=yes polygon, other area as no - if tags["area"] then - return tags["area"] == "yes" and 1 or 0 - end - - -- Search through object's tags - for k, v in pairs(tags) do - -- Check if it has a polygon key and not a linestring override, or a polygon k=v - for _, ptag in ipairs(polygon_keys) do - if k == ptag and v ~= "no" and not (linestring_values[k] and linestring_values[k][v]) then - return 1 - end - end - - if (polygon_values[k] and polygon_values[k][v]) then - return 1 - end - end - return 0 -end - -function is_in (needle, haystack) - for index, value in ipairs (haystack) do - if value == needle then - return true - end - end - return false -end - ---- Normalizes layer tags --- @param v The layer tag value --- @return An integer for the layer tag -function layer (v) - return v and string.find(v, "^-?%d+$") and tonumber(v) < 100 and tonumber(v) > -100 and v or nil -end diff --git a/openstreetmap-carto.style b/openstreetmap-carto.style deleted file mode 100644 index 0a30add22d..0000000000 --- a/openstreetmap-carto.style +++ /dev/null @@ -1,55 +0,0 @@ -# This is the osm2pgsql .style file for openstreetmap-carto. -# It is intended to be used with openstreetmap-carto.lua and osm2pgsql Lua -# transforms. Full usage details are in INSTALL.md -# Among things, this means that the linear vs polygon distinction in this file -# doesn't matter, because that is set in the Lua and this file is only used for -# column names and types. - -# OsmType Tag DataType Flags -node,way access text linear -node,way addr:housename text linear -node,way addr:housenumber text linear -way addr:interpolation text linear -node,way admin_level text linear -node,way aerialway text linear -node,way aeroway text polygon -node,way amenity text polygon -node,way barrier text linear -way bicycle text linear -way bridge text linear -node,way boundary text linear -node,way building text polygon -way construction text linear -way covered text linear -way foot text linear -node,way highway text linear -node,way historic text polygon -way horse text linear -node,way junction text linear -node,way landuse text polygon -node,way layer int4 linear -node,way leisure text polygon -node,way lock text linear -node,way man_made text polygon -node,way military text polygon -node,way name text linear -node,way natural text polygon -node,way oneway text linear -node,way place text polygon -node,way power text polygon -node,way railway text linear -node,way ref text linear -node,way religion text linear -way route text linear -way service text linear -node,way shop text polygon -way surface text linear -node,way tourism text polygon -way tracktype text linear -way tunnel text linear -node,way water text polygon -node,way waterway text polygon -way way_area real linear # This is calculated during import - -# Columns defined in openstreetmap-carto.lua file -way z_order int4 linear diff --git a/scripts/docker-startup.sh b/scripts/docker-startup.sh index c3177ee9d1..6b60fe7b58 100644 --- a/scripts/docker-startup.sh +++ b/scripts/docker-startup.sh @@ -43,13 +43,11 @@ EOF osm2pgsql \ --cache $OSM2PGSQL_CACHE \ --number-processes $OSM2PGSQL_NUMPROC \ - --hstore \ - --multi-geometry \ --database gis \ --slim \ --drop \ - --style openstreetmap-carto.style \ - --tag-transform-script openstreetmap-carto.lua \ + --output flex \ + --style openstreetmap-carto-flex.lua \ $OSM2PGSQL_DATAFILE # Downloading and importing needed shapefiles diff --git a/scripts/lua/README.md b/scripts/lua/README.md deleted file mode 100644 index b5c3a4ef4e..0000000000 --- a/scripts/lua/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Lua helper scripts # - -These scripts are for developing, testing, and profiling the [Lua tag transform](../../openstreetmap-carto.lua). There is a symlink to the transform in this directory so it can be `require`d by other files. - -They are not necessary for map rendering or most development. diff --git a/scripts/lua/openstreetmap-carto.lua b/scripts/lua/openstreetmap-carto.lua deleted file mode 120000 index ba19fbe902..0000000000 --- a/scripts/lua/openstreetmap-carto.lua +++ /dev/null @@ -1 +0,0 @@ -../../openstreetmap-carto.lua \ No newline at end of file diff --git a/scripts/lua/test.lua b/scripts/lua/test.lua deleted file mode 100644 index 9f7962fc48..0000000000 --- a/scripts/lua/test.lua +++ /dev/null @@ -1,188 +0,0 @@ ---[[ -This file is part of OpenStreetMap Carto and used for validating the Lua tag transforms. - -Run it with lua test.lua -]] - -require ("openstreetmap-carto") - ---- compare two tables. --- @param t1 A table --- @param t2 A table --- @return true or false -function equaltables (t1,t2) - for k, v in pairs(t1) do - if t2[k] ~= v then return false end - end - for k, v in pairs(t2) do - if t1[k] ~= v then return false end - end - return true -end - -print("TESTING: z_order") - -assert(z_order({}) == nil, "test failed: no tags") -assert(z_order({foo="bar"}) == nil, "test failed: other tags") -assert(z_order({highway="motorway"}) == 380 , "test failed: motorway") -assert(z_order({highway="motorway", railway="rail"}) == 440 , "test failed: motorway + rail") - -assert(z_order({highway="motorway"}) > z_order({highway="motorway_link"}) , "test failed: motorway_link") -assert(z_order({highway="trunk"}) > z_order({highway="trunk_link"}) , "test failed: trunk_link") -assert(z_order({highway="primary"}) > z_order({highway="primary_link"}) , "test failed: primary_link") -assert(z_order({highway="secondary"}) > z_order({highway="secondary_link"}) , "test failed: secondary_link") -assert(z_order({highway="tertiary"}) > z_order({highway="tertiary_link"}) , "test failed: tertiary_link") - -assert(z_order({highway="motorway"}) > z_order({highway="trunk"}) , "test failed: motorway > trunk") -assert(z_order({highway="trunk"}) > z_order({highway="primary"}) , "test failed: trunk > primary") -assert(z_order({highway="primary"}) > z_order({highway="secondary"}) , "test failed: primary > secondary") -assert(z_order({highway="secondary"}) > z_order({highway="tertiary"}) , "test failed: secondary > tertiary") - -assert(z_order({highway="construction"}) == 33 , "test failed: highway=construction") -assert(z_order({highway="construction", construction="motorway"}) == 38 , "test failed: highway=construction construction=motorway") -assert(z_order({highway="construction", construction="motorway", railway="rail"}) == 440, "test failed: construction motorway + rail") -assert(z_order({highway="construction", construction="service"}) == 15 , "test failed: highway=construction construction=service") - -assert(z_order({highway="construction", construction="foo"}) == 33 , "test failed: highway=construction construction=foo") -assert(z_order({highway="motorway", construction="service"}) == 380 , "test failed: highway=construction + construction=service") - -print("TESTING: roads") -assert(roads({}) == 0, "test failed: no tags") -assert(roads({foo="bar"}) == 0, "test failed: other tags") -assert(roads({highway="motorway"}) == 1, "test failed: motorway") -assert(roads({railway="rail"}) == 1, "test failed: rail") -assert(roads({highway="residential", railway="rail"}) == 1, "test failed: rail+residential") -assert(roads({railway="turntable"}) == 0, "test failed: rail=turntable") -assert(roads({railway="rail", service="spur"}) == 0, "test failed: rail SSY") -assert(roads({railway="rail", service="main"}) == 1, "test failed: rail non-SSY") -assert(roads({boundary="administrative"}) == 1, "test failed: boundary administrative") - -print("TESTING: isarea") -assert(isarea({}) == 0, "test failed: no tags") -assert(isarea({foo = "bar"}) == 0, "test failed: random tag") -assert(isarea({area = "yes"}) == 1, "test failed: explicit area") -assert(isarea({area = "no"}) == 0, "test failed: explicit not area") -assert(isarea({area = "no", landuse = "forest"}) == 0, "test failed: explicit not area with polygon tag") -assert(isarea({leisure = "track"}) == 0, "test failed: leisure=track") -assert(isarea({area = "yes", leisure = "track"}) == 1, "test failed: leisure=track with area tag") -assert(isarea({waterway = "river"}) == 0, "test failed: river") -assert(isarea({waterway = "riverbank"}) == 1, "test failed: river") -assert(isarea({highway = "services"}) == 1, "test failed: river") -assert(isarea({natural="cliff"}) == 0, "test failed: cliff") -- issue #3084 -assert(isarea({building = "no"}) == 0, "test failed: building=no") -assert(isarea({building = "no", area = "yes"}) == 1, "test failed: building=no with area tag") -assert(isarea({building = "no", landuse = "forest"}) == 1, "test failed: building=no with other area tag") - -print("TESTING: filter_tags_generic") -assert(({filter_tags_generic({})})[1] == 1, "Untagged filter") -assert(equaltables(({filter_tags_generic({})})[2], {}), "Untagged tags") -assert(({filter_tags_generic({note="foo"})})[1] == 1, "deleted filter") -assert(equaltables(({filter_tags_generic({note="foo"})})[2], {}), "deleted tags") -assert(({filter_tags_generic({foo="bar"})})[1] == 0, "single tag filter") -assert(equaltables(({filter_tags_generic({foo="bar"})})[2], {foo="bar"}), "single tag tags") -assert(({filter_tags_generic({foo="bar", note="baz"})})[1] == 0, "tag + deleted tag filter") -assert(equaltables(({filter_tags_generic({foo="bar", note="baz"})})[2], {foo="bar"}), "tag + deleted tags") -assert(({filter_tags_generic({["note:xx"]="foo"})})[1] == 1, "wildcard deleted filter") -assert(equaltables(({filter_tags_generic({["note:xx"]="foo"})})[2], {}), "wildcard deleted tags") -assert(({filter_tags_generic({["note:xx"]="foo", foo="bar"})})[1] == 0, "wildcard deleted + tag filter") -assert(equaltables(({filter_tags_generic({["note:xx"]="foo", foo="bar"})})[2], {foo="bar"}), "wildcard deleted + tag tags") - -assert(({filter_tags_generic({["foo:note:xx"]="foo"})})[1] == 0, "prefix later in tag filter") -assert(equaltables(({filter_tags_generic({["foo:note:xx"]="foo"})})[2], {["foo:note:xx"]="foo"}), "prefix later in tag tags") - -print("TESTING: filter_tags_relation_member") - ---- Tests filter_tags_relation_member against expected values --- @param keyvalues OSM tags, after processing by relation transform --- @param keyvaluemembers OSM tags of relation members, after processing by way transform --- @param filter expected filter result --- @param cols expected cols result --- @param member_superseded expected member_superseded result --- @param boundary expected boundary result --- @param polygon expected polygon result --- @param roads expected roads result -local function check_rel_member(keyvalues, keyvaluemembers, filter, cols, member_superseded, boundary, polygon, roads) - - local i = 0 - for _ in pairs(keyvaluemembers) do - i = i + 1 - end - - local actual_filter, actual_cols, actual_member_superseded, actual_boundary, actual_polygon, actual_roads - = filter_tags_relation_member(keyvalues, keyvaluemembers, nil, i) - - if actual_filter ~= filter then - print("filter mismatch") - return false - end - if not equaltables(actual_cols, cols) then - print("cols mismatch") - return false - end - if not equaltables(actual_member_superseded, member_superseded) then - print("member_superseded mismatch, actual table was") - for i, v in ipairs(actual_member_superseded) do - print(i, v) - end - return false - end - if actual_boundary ~= boundary then - print("boundary mismatch") - return false - end - if actual_polygon ~= polygon then - print("polygon mismatch") - return false - end - if actual_roads ~= roads then - print("roads mismatch") - return false - end - return true -end - -assert(check_rel_member({}, {}, 1, {}, {}, 0, 0, 0), "test failed: untagged memberless relation") -assert(check_rel_member({}, {{}}, 1, {}, {0}, 0, 0, 0), "test failed: untagged relation") - -assert(check_rel_member({type="multipolygon"}, {{}}, 1, {}, {0}, 0, 0, 0), - "test failed: untagged MP") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{}}, 0, {foo="bar"}, {0}, 0, 1, 0), - "test failed: MP with tag") - --- New-style MPs -assert(check_rel_member({type="multipolygon", foo="bar"}, {{},{}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, two ways") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{baz="qax"}}, 0, {foo="bar"}, {0}, 0, 1, 0), - "test failed: MP with tag, way with different tag") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{baz="qax"}, {}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, way with different tag + untagged way") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{foo="bar"}}, 0, {foo="bar"}, {0}, 0, 1, 0), - "test failed: MP with tag, way with same tag") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{foo="bar"},{}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, way with same tag + untagged way") -assert(check_rel_member({type="multipolygon", foo="bar"}, {{foo="bar"}, {baz="qax"}}, 0, {foo="bar"}, {0,0}, 0, 1, 0), - "test failed: MP with tag, way with same tag") - --- Old-style MPs -assert(check_rel_member({type="multipolygon"}, {{foo="bar"}}, 1, {}, {0}, 0, 0, 0), - "test failed: MP w/o tag, way with tag") -assert(check_rel_member({type="multipolygon"}, {{foo="bar"}, {}}, 1, {}, {0,0}, 0, 0, 0), - "test failed: MP w/o tag, way with tag + untagged way") -assert(check_rel_member({type="multipolygon"}, {{foo="bar"}, {baz="qax"}}, 1, {}, {0,0}, 0, 0, 0), - "test failed: MP w/o tag, way with tag + way with other tag") - --- Boundary relations -assert(check_rel_member({type="boundary"}, {{}}, 1, {}, {0}, 0, 0, 0), - "test failed: untagged boundary") -assert(check_rel_member({type="boundary", boundary="administrative"}, {{}}, 0, {boundary="administrative"}, {0}, 1, 0, 1), - "test failed: untagged boundary") -assert(check_rel_member({type="boundary", boundary="administrative"}, {{}}, 0, {boundary="administrative"}, {0}, 1, 0, 1), - "test failed: untagged boundary") -assert(check_rel_member({type="boundary", boundary="administrative"}, {{foo="bar"}}, 0, {boundary="administrative"}, {0}, 1, 0, 1), - "test failed: untagged boundary, tagged way") - --- Route relations -assert(check_rel_member({type="route"}, {{}}, 1, {}, {0}, 0, 0, 0), - "test failed: untagged route") -assert(check_rel_member({type="route", route="road"}, {{}}, 0, {route="road"}, {0}, 1, 0, 0), - "test failed: tagged route")