Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Join countries and regions to Natural Earth for min and max zoom #1535

Merged
merged 6 commits into from
Jun 15, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions data/apply-schema-update.sql
Original file line number Diff line number Diff line change
@@ -121,6 +121,16 @@ PERFORM add_column_if_not_exists('ne_10m_land', 'mz_label_placement', 'geometry(
PERFORM add_column_if_not_exists('ne_50m_urban_areas', 'mz_label_placement', 'geometry(Geometry, 3857)');
PERFORM add_column_if_not_exists('ne_10m_urban_areas', 'mz_label_placement', 'geometry(Geometry, 3857)');

-- NOTE: the absence of:
--
-- * ne_10m_admin_0_countries
-- * ne_10m_admin_0_map_units
-- * ne_10m_admin_1_states_provinces
--
-- although we do import the _boundaries_ variants of some of these, we only use
-- the polygon variants above to join on the wikidata ID, we don't draw or label
-- them directly.

END$$;

DROP FUNCTION add_column_if_not_exists(text, text, text);
4 changes: 4 additions & 0 deletions data/apply-updates-non-planet-tables.sql
Original file line number Diff line number Diff line change
@@ -187,3 +187,7 @@ CREATE INDEX ne_10m_land_way_area_index ON ne_10m_land(way_area);
UPDATE land_polygons SET way_area=ST_Area(the_geom) WHERE the_geom IS NOT NULL;
CREATE INDEX land_polygons_wayarea_index ON land_polygons(way_area);

-- we look up min_label, max_label by wikidata ID.
CREATE INDEX ne_10m_admin_0_countries_wikidata_index ON ne_10m_admin_0_countries(wikidataid);
CREATE INDEX ne_10m_admin_0_map_units_wikidata_index ON ne_10m_admin_0_map_units(wikidataid);
CREATE INDEX ne_10m_admin_1_states_provinces_wikidata_index ON ne_10m_admin_1_states_provinces(wikidataid);
14 changes: 13 additions & 1 deletion data/assets.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
bucket: tilezen-assets
datestamp: 20180612
datestamp: 20180614

shapefiles:

@@ -129,3 +129,15 @@ shapefiles:
- name: ne_10m_coastline
url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/physical/ne_10m_coastline.zip
prj: 43267

- name: ne_10m_admin_0_countries
url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_countries.zip
prj: 4326

- name: ne_10m_admin_0_map_units
url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_0_map_units.zip
prj: 4326

- name: ne_10m_admin_1_states_provinces
url: https://www.naturalearthdata.com/http//www.naturalearthdata.com/download/10m/cultural/ne_10m_admin_1_states_provinces.zip
prj: 4326
51 changes: 50 additions & 1 deletion data/functions.sql
Original file line number Diff line number Diff line change
@@ -700,7 +700,7 @@ DECLARE
decimal_matches text[] :=
regexp_matches(txt, '([0-9]+(\.[0-9]*)?) *(mi|km|m|nmi|ft)');
imperial_matches text[] :=
regexp_matches(txt, E'([0-9]+(\\.[0-9]*)?)\' *(([0-9]+)")?');
regexp_matches(txt, E'([0-9]+(\\.[0-9]*)?)\x27 *(([0-9]+)")?');
numeric_matches text[] :=
regexp_matches(txt, '([0-9]+(\.[0-9]*)?)');
BEGIN
@@ -980,3 +980,52 @@ BEGIN
RETURN trim(leading 'SH' from label);
END
$$ LANGUAGE plpgsql IMMUTABLE;

-- returns a JSONB object containing __ne_min_zoom and __ne_max_zoom set to the
-- label min and max zoom of any matching row from the Natural Earth countries,
-- map units and states/provinces themes.
CREATE OR REPLACE FUNCTION tz_get_ne_min_max_zoom(wikidata_id TEXT)
RETURNS JSONB AS $$
DECLARE
min_zoom REAL;
max_zoom REAL;
BEGIN
IF wikidata_id IS NULL THEN
RETURN '{}'::jsonb;
END IF;

-- first, try the countries table
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_0_countries c
WHERE c.wikidataid = wikidata_id;

-- if that fails, try map_units (which contains some sub-country but super-
-- state level stuff such as England, Scotland and Wales).
IF NOT FOUND THEN
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_0_map_units mu
WHERE mu.wikidataid = wikidata_id;
END IF;

-- finally, try states and provinces
IF NOT FOUND THEN
SELECT
min_label, max_label INTO min_zoom, max_zoom
FROM ne_10m_admin_1_states_provinces sp
WHERE sp.wikidataid = wikidata_id;
END IF;

-- return an empty JSONB rather than null, so that it can be safely
-- concatenated with whatever other JSONB rather than needing a check for
-- null.
IF NOT FOUND THEN
RETURN '{}'::jsonb;
END IF;
RETURN jsonb_build_object(
'__ne_min_zoom', min_zoom,
'__ne_max_zoom', max_zoom
);
END
$$ LANGUAGE plpgsql STABLE;
4 changes: 2 additions & 2 deletions integration-test/837-no-country-label-low-zoom.py
Original file line number Diff line number Diff line change
@@ -9,5 +9,5 @@ def test_no_country_label_at_low_zoom(self):
self.assert_no_matching_feature(
0, 0, 0, 'places', {'kind': 'country'})

self.assert_no_matching_feature(
1, 0, 0, 'places', {'kind': 'country'})
# see integration-test/977-min-zoom-from-ne-join.py -- we now get this
# information from joining on the Natural Earth countries table.
145 changes: 145 additions & 0 deletions integration-test/977-min-zoom-from-ne-join.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
# -*- coding: utf-8 -*-
from . import FixtureTest


class MinZoomFromNETest(FixtureTest):

def setUp(self):
import dsl

super(MinZoomFromNETest, self).setUp()

self.lon, self.lat = (-3.2765753, 54.7023545)

self.generate_fixtures(
# https://www.openstreetmap.org/node/838090640
dsl.point(838090640, (self.lon, self.lat), {
'name': u'United Kingdom',
'place': u'country',
'population': u'61792000',
'source': u'openstreetmap.org',
'wikidata': u'Q145',
'wikipedia': u'de:United Kingdom', # LOL, de:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:)

# NOTE: these aren't in the data from OSM, but are joined at
# database query time from the Natural Earth data.
'__ne_min_zoom': 1.7,
'__ne_max_zoom': 6.7,
}),
)

def test_uk_should_show_up_zooms_1_to_6(self):
from tilequeue.tile import deg2num
# should show up in zooms within the range 1-6

for zoom in xrange(1, 6):
x, y = deg2num(self.lat, self.lon, zoom)
self.assert_has_feature(
zoom, x, y, 'places', {
'id': 838090640,
'min_zoom': 1.7,
'max_zoom': 6.7,
})

def test_uk_should_not_show_up_zoom_0(self):
# shouldn't be in the zoom 0 tile because min_zoom >= 1
self.assert_no_matching_feature(
0, 0, 0, 'places', {'id': 838090640})

def test_uk_should_not_show_up_zoom_7(self):
# shouldn't be in the zoom 0 tile because max_zoom < 7
from tilequeue.tile import deg2num

zoom = 7
x, y = deg2num(self.lat, self.lon, zoom)
self.assert_no_matching_feature(
zoom, x, y, 'places', {'id': 838090640})


class MinZoomFromAdminAreaBasedDefault(FixtureTest):

def test_united_kingdom(self):
# in the absence of data joined from NE, we should fall back to a
# default based on the country that the label point is in.
import dsl
from tilequeue.tile import deg2num

lon, lat = (-3.2765753, 54.7023545)
z = 5
x, y = deg2num(lat, lon, z)

self.generate_fixtures(
dsl.is_in('GB', z, x, y),
# https://www.openstreetmap.org/node/838090640
dsl.point(838090640, (lon, lat), {
'name': u'United Kingdom',
'place': u'country',
'population': u'61792000',
'source': u'openstreetmap.org',
'wikidata': u'Q145',
'wikipedia': u'de:United Kingdom', # LOL, de:
}),
)

self.assert_has_feature(
z, x, y, 'places', {
'id': 838090640,
'min_zoom': 1.7,
'max_zoom': 6.7,
})

def test_ne_min_zoom_should_override_default(self):
import dsl
from tilequeue.tile import deg2num

lon, lat = (-3.2765753, 54.7023545)
z = 5
x, y = deg2num(lat, lon, z)

self.generate_fixtures(
dsl.is_in('GB', z, x, y),
# https://www.openstreetmap.org/node/838090640
dsl.point(838090640, (lon, lat), {
'name': u'United Kingdom',
'place': u'country',
'population': u'61792000',
'source': u'openstreetmap.org',
'wikidata': u'Q145',
'wikipedia': u'de:United Kingdom', # LOL, de:
# NE joins should override defaults from location
'__ne_min_zoom': 0,
'__ne_max_zoom': 16,
}),
)

self.assert_has_feature(
z, x, y, 'places', {
'id': 838090640,
'min_zoom': 0,
'max_zoom': 16,
})

def test_wales(self):
# wales is a country within the UK, but mapped as place=state.
# should get a fallback from the states_provinces spreadsheet.
import dsl

z, x, y = (10, 501, 336)

self.generate_fixtures(
dsl.is_in('GB', z, x, y),
# https://www.openstreetmap.org/node/2642288017
dsl.point(2642288017, (-3.73893, 52.2928116), {
'is_in': u'United Kingdom, Europe',
'name': u'Wales',
'note': u'geographical centre of Wales',
'place': u'state',
'source': u'openstreetmap.org',
}),
)

self.assert_has_feature(
z, x, y, 'places', {
'id': 2642288017,
'min_zoom': 10,
'max_zoom': 11,
})
43 changes: 43 additions & 0 deletions queries.yaml
Original file line number Diff line number Diff line change
@@ -549,6 +549,45 @@ post_process:
ZM: true # Zambia
ZW: true # Zimbabwe

# cut places with admin_areas to put country_code attributes on country,
# state and province points. this is used to backfill information such as
# min and max zoom based on country defaults.
- fn: vectordatasource.transform.point_in_country_logic
resources:
logic_table:
type: file
init_fn: vectordatasource.transform.YAMLToDict
path: spreadsheets/min_zoom/country.yaml
params:
layer: places
country_layer: admin_areas
country_code_attr: iso_code
output_attrs:
- min_zoom
- max_zoom
where: kind == 'country'

- fn: vectordatasource.transform.point_in_country_logic
resources:
logic_table:
type: file
init_fn: vectordatasource.transform.YAMLToDict
path: spreadsheets/min_zoom/state_province.yaml
params:
layer: places
country_layer: admin_areas
country_code_attr: iso_code
output_attrs:
- min_zoom
- max_zoom
where: kind == 'region'

# IMPORTANT! do this _after_ the YAMLToDict default stuff, otherwise the
# default will overwrite the value fron Natural Earth.
- fn: vectordatasource.transform.tags_set_ne_min_max_zoom
params:
layer: places

- fn: vectordatasource.transform.overlap
params:
base_layer: buildings
@@ -1053,3 +1092,7 @@ post_process:
white: [255, 255, 255]
yellow: [255, 255, 0]
yellowgreen: [154, 205, 50]

- fn: vectordatasource.transform.max_zoom_filter
params:
layers: [places]
3 changes: 2 additions & 1 deletion queries/planet_osm_point.jinja2
Original file line number Diff line number Diff line change
@@ -26,7 +26,8 @@ SELECT
CASE WHEN mz_places_min_zoom IS NOT NULL
THEN jsonb_build_object(
'min_zoom', mz_places_min_zoom
)
) ||
tz_get_ne_min_max_zoom(tags->'wikidata')
END AS __places_properties__,

CASE WHEN mz_poi_min_zoom IS NOT NULL
Loading