Merge pull request gravitystorm#2100 from pnorman/indexes

Add custom index information

INSTALL.md

@@ -1,5 +1,6 @@
 # Installation
 
+## OpenStreetMap data
 You need OpenStreetMap data loaded into a PostGIS database (see below for [dependencies](#dependencies)). These stylesheets currently work only with the osm2pgsql defaults (i.e. database name is `gis`, table names are `planet_osm_point`, etc).
 
 It's probably easiest to grab an PBF of OSM data from [Mapzen](https://mapzen.com/metro-extracts/) or [geofabrik](http://download.geofabrik.de/). Once you've set up your PostGIS database, import with osm2pgsql:
@@ -10,6 +11,13 @@ osm2pgsql -d gis ~/path/to/data.osm.pbf --style openstreetmap-carto.style
 
 You can find a more detailed guide to setting up a database and loading data with osm2pgsql at [switch2osm.org](http://switch2osm.org/loading-osm-data/).
 
+### Custom indexes
+Custom indexes are not required, but will speed up rendering, particularly for full planet databases, heavy load, or other production environments. They will not be as helpful with development using small extracts.
+
+```
+psql -d gis -f indexes.sql
+```
+
 Additionally you need some shapefiles.
 
 ## Scripted download

indexes.sql

@@ -0,0 +1,37 @@
+-- These are optional but suggested indexes for rendering OpenStreetMap Carto
+-- with a full planet database.
+-- This file is generated with scripts/indexes.py
+
+CREATE INDEX planet_osm_roads_admin
+  ON planet_osm_roads USING GIST (way)
+  WHERE boundary = 'administrative';
+CREATE INDEX planet_osm_roads_roads_ref
+  ON planet_osm_roads USING GIST (way)
+  WHERE highway IS NOT NULL AND ref IS NOT NULL;
+CREATE INDEX planet_osm_roads_admin_low
+  ON planet_osm_roads USING GIST (way)
+  WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
+CREATE INDEX planet_osm_line_ferry
+  ON planet_osm_line USING GIST (way)
+  WHERE route = 'ferry';
+CREATE INDEX planet_osm_line_river
+  ON planet_osm_line USING GIST (way)
+  WHERE waterway = 'river';
+CREATE INDEX planet_osm_line_name
+  ON planet_osm_line USING GIST (way)
+  WHERE name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_military
+  ON planet_osm_polygon USING GIST (way)
+  WHERE landuse = 'military';
+CREATE INDEX planet_osm_polygon_nobuilding
+  ON planet_osm_polygon USING GIST (way)
+  WHERE building IS NULL;
+CREATE INDEX planet_osm_polygon_name
+  ON planet_osm_polygon USING GIST (way)
+  WHERE name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_way_area_z6
+  ON planet_osm_polygon USING GIST (way)
+  WHERE way_area > 59750;
+CREATE INDEX planet_osm_point_place
+  ON planet_osm_point USING GIST (way)
+  WHERE place IS NOT NULL AND name IS NOT NULL;

indexes.yml

@@ -0,0 +1,34 @@
+# A goal with the indexes is to have them general-purpose enough to not need
+# frequent changing with stylesheet changes, but to be usable with many
+# versions, and potentially other styles.
+
+point:
+  # The point table is small, but so are the partial indexes
+  place:
+    where: place IS NOT NULL AND name IS NOT NULL
+line:
+  name:
+    where: name IS NOT NULL
+  ferry:
+    where: route = 'ferry'
+  river:
+    where: waterway = 'river'
+polygon:
+  # The polygon table is by far the largest, and generally the slowest
+  name:
+    where: name IS NOT NULL
+  nobuilding:
+    where: building IS NULL
+  military:
+    where: landuse = 'military'
+  way_area_z6:
+    where: way_area > 59750
+roads:
+  # The roads table only has a subset of data, so it's just got some low-zoom
+  # indexes and some fairly selective ones for high zoom
+  admin_low:
+    where: boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4')
+  admin:
+    where: boundary = 'administrative'
+  roads_ref:
+    where: highway IS NOT NULL AND ref IS NOT NULL

scripts/indexes.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+# This script takes a YAML file defining indexes and creates SQL statements
+# There are a number of options for concurrent index creation, recreating the
+# osm2pgsql-built indexes, fillfactors, and other settings to give full control
+# of the resulting statements
+# indexes.sql is created by this script, with the default options
+
+from __future__ import print_function
+import argparse, sys, os, yaml
+
+def index_statement(table, name, conditions=None, concurrent=False,notexist=False, fillfactor=None):
+    options = ' CONCURRENTLY' if concurrent else ''
+    options += ' IF NOT EXISTS' if notexist else ''
+    storage = '' if fillfactor is None else '\n  WITH (fillfactor={})'.format(fillfactor)
+    where = '' if conditions is None else '\n  WHERE {}'.format(conditions)
+    return ('CREATE INDEX{options} {table}_{name}\n' +
+            '  ON {table} USING GIST (way)' +
+            '{storage}' +
+            '{where};\n').format(table="planet_osm_"+table, name=name,
+                storage=storage, options=options, where=where)
+
+def parse(cb):
+    with open(os.path.join(os.path.dirname(__file__), '../indexes.yml')) as yaml_file:
+        indexes = yaml.safe_load(yaml_file)
+
+    for table, data in indexes.iteritems():
+        for name, definition in data.iteritems():
+            cb(table, name, definition["where"])
+
+# The same as parse, but for osm2pgsql-built indexes
+def osm2pgsql_parse(cb):
+    cb('point', 'index', None)
+    cb('line', 'index', None)
+    cb('polygon', 'index', None)
+    cb('roads', 'index', None)
+
+parser = argparse.ArgumentParser(description='Generates custom index statements')
+parser.add_argument('--concurrent', dest='concurrent', help='Generate indexes CONCURRENTLY', action='store_true', default=False)
+parser.add_argument('--fillfactor', help='Custom fillfactor to use')
+parser.add_argument('--notexist', help='Use IF NOT EXISTS (requires 9.5)', action='store_true', default=False)
+parser.add_argument('--osm2pgsql', help='Include indexes normally built by osm2pgsql', action='store_true', default=False)
+parser.add_argument('--reindex', help='Rebuild existing indexes', action='store_true', default=False)
+args = parser.parse_args()
+
+def cb (table, name, where):
+    print(index_statement(table, name, where, args.concurrent, args.notexist, args.fillfactor), end='')
+
+def reindex_cb(table, name, where):
+    if not args.concurrent:
+        print('REINDEX planet_osm_{table}_{name};'.format(table=table, name=name))
+    else:
+        # Rebuilding indexes concurently requires making a new index, dropping the old one, and renaming.
+        print('ALTER INDEX planet_osm_{table}_{name} RENAME TO planet_osm_{table}_{name}_old;'.format(table=table, name=name))
+        cb(table, name, where)
+        print('DROP INDEX planet_osm_{table}_{name}_old;\n'.format(table=table, name=name))
+
+print(('-- These are optional but suggested indexes for rendering OpenStreetMap Carto\n'+
+      '-- with a full planet database.\n'
+      '-- This file is generated with {}\n').format(' '.join(sys.argv)))
+
+if not args.reindex:
+    parse(cb)
+else:
+    parse(reindex_cb)
+
+if args.osm2pgsql:
+    print('\n-- These indexes are normally built by osm2pgsql')
+    if not args.reindex:
+        osm2pgsql_parse(cb)
+    else:
+        osm2pgsql_parse(reindex_cb)