Merge pull request #4092 from pnorman/shapefiles_in_db

Move external shapefiles to tables in the DB

.travis.yml

@@ -21,8 +21,6 @@ env:
 install:
   - npm install carto@$CARTO
   - pip3 install --user colormath
-  - mkdir -p data/simplified-water-polygons-split-3857 data/ne_110m_admin_0_boundary_lines_land data/water-polygons-split-3857
-  - touch data/simplified-water-polygons-split-3857/simplified_water_polygons.shp data/ne_110m_admin_0_boundary_lines_land/ne_110m_admin_0_boundary_lines_land.shp data/water-polygons-split-3857/water_polygons.shp
   - createdb -w -E utf8 -U postgres gis && psql -Xq -d gis -U postgres -w -c "CREATE EXTENSION postgis; CREATE EXTENSION hstore;"
 script:
   # We're using pipes in the checks, so fail if any part fails

DOCKER.md

@@ -61,7 +61,7 @@ Depending on your machine and the size of the extract the import can take a whil
 ## Test rendering
 
 After you have the necessary data available you can start Kosmtik to produce a test rendering. For that you run `docker-compose up kosmtik` in the openstreetmap-carto directory. This starts a container with Kosmtik and also starts the PostgreSQL database container if it is not already running. The Kosmtik container is built the first time you run that command if it not exists.
-At startup of the container the script `scripts/docker-startup.sh` is invoked which downloads necessary shapefiles with `scripts/get-shapefiles.py` (if they are not already present) and indexes them. It afterwards runs Kosmtik. If you have to customize anything, you can do so in the script. The Kosmtik config file can be found in `.kosmtik-config.yml`.
+At startup of the container the script `scripts/docker-startup.sh` is invoked which downloads necessary shapefiles with `scripts/get-external-data.py` (if they are not already present). It afterwards runs Kosmtik. If you have to customize anything, you can do so in the script. The Kosmtik config file can be found in `.kosmtik-config.yml`.
 If you want to have a [local configuration](https://github.com/kosmtik/kosmtik#local-config) for our `project.mml` you can place a `localconfig.js` or `localconfig.json` file into the openstreetmap-carto directory.
 
 The shapefile data that is downloaded is owned by the user with UID 1000. If you have another default user id on your system, consider changing the line `USER 1000` in the file `Dockerfile`.

Dockerfile.import

@@ -11,7 +11,8 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \
     --recv A438A16C88C6BE41CB1616B8D57F48750AC4F2CB
 
 RUN apt-get update && apt-get install --no-install-recommends -y \
-    osm2pgsql postgresql-client && rm -rf /var/lib/apt/lists/*
+    osm2pgsql gdal-bin python3-psycopg2 python3-yaml \
+    python3-requests postgresql-client && rm -rf /var/lib/apt/lists/*
 
 ADD openstreetmap-carto.style /
 

INSTALL.md

@@ -36,28 +36,14 @@ psql -d gis -f indexes.sql
 ## Scripted download
 Some features are rendered using preprocessed shapefiles.
 
-To obtain them you can run the following script.
+To download them and import them into the database you can run the following script
 
 ```
-scripts/get-shapefiles.py
+scripts/get-external-data.py
 ```
 
 This script downloads necessary files, generates and populates the *data* directory with all needed shapefiles, including indexing them through *shapeindex*.
 
-## Manual download
-
-You can also download them manually at the following paths:
-
-* [`simplified_water_polygons.shp`](https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip) (updated daily)
-* [`ne_110m_admin_0_boundary_lines_land.shp`](http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip)
-* [`water_polygons.shp`](https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip) (updated daily)
-* [`icesheet_polygons.shp`](https://osmdata.openstreetmap.de/download/antarctica-icesheet-polygons-3857.zip)
-* [`icesheet_outlines.shp`](https://osmdata.openstreetmap.de/download/antarctica-icesheet-outlines-3857.zip)
-
-The repeated www.naturalearthdata.com in the Natural Earth shapefiles is correct.
-
-Put these shapefiles at `path/to/openstreetmap-carto/data`.
-
 ## Fonts
 The stylesheet uses Noto, an openly licensed font family from Google with support for multiple scripts. The stylesheet uses Noto's "Sans" style where available. If not available, this stylesheet uses another appropriate style of the Noto family. The "UI" version is used where available, with its vertical metrics which fit better with Latin text.
 
@@ -116,8 +102,8 @@ To display any map a database containing OpenStreetMap data and some utilities a
 * [PostgreSQL](https://www.postgresql.org/)
 * [PostGIS](https://postgis.net/)
 * [osm2pgsql](https://github.com/openstreetmap/osm2pgsql#installing) to [import your data](https://switch2osm.org/loading-osm-data/) into a PostGIS database
-* `curl` and `unzip` for downloading and decompressing files
-* shapeindex (a companion utility to Mapnik found in the `mapnik-utils` package) for indexing downloaded shapefiles
+* Python 3 with the psycopg2, yaml, and requests libraries (`python3-psycopg2` `python3-yaml` `python3-requests` packages on Debian-derived systems)
+* `ogr2ogr` for loading shapefiles into the database (`gdal-bin` on Debian-derived systems)
 
 ### Optional development dependencies
 

external-data.yml

@@ -0,0 +1,81 @@
+settings:
+  temp_schema: loading
+  schema: public
+  data_dir: data
+  database: gis
+  metadata_table: external_data
+sources:
+  simplified_water_polygons:
+    # The type of file this source is
+    type: shp
+    # Where to get it
+    url: https://osmdata.openstreetmap.de/download/simplified-water-polygons-split-3857.zip
+    # The location within the archive
+    file: simplified-water-polygons-split-3857/simplified_water_polygons.shp
+    archive:
+      format: zip
+      # Files to extract from the archive
+      files:
+        - simplified-water-polygons-split-3857/simplified_water_polygons.cpg
+        - simplified-water-polygons-split-3857/simplified_water_polygons.dbf
+        - simplified-water-polygons-split-3857/simplified_water_polygons.prj
+        - simplified-water-polygons-split-3857/simplified_water_polygons.shp
+        - simplified-water-polygons-split-3857/simplified_water_polygons.shx
+  water_polygons:
+    type: shp
+    url: https://osmdata.openstreetmap.de/download/water-polygons-split-3857.zip
+    file: water-polygons-split-3857/water_polygons.shp
+    archive:
+      format: zip
+      files:
+        - water-polygons-split-3857/water_polygons.cpg
+        - water-polygons-split-3857/water_polygons.dbf
+        - water-polygons-split-3857/water_polygons.prj
+        - water-polygons-split-3857/water_polygons.shp
+        - water-polygons-split-3857/water_polygons.shx
+  icesheet_polygons:
+    type: shp
+    url: https://osmdata.openstreetmap.de/download/antarctica-icesheet-polygons-3857.zip
+    file: antarctica-icesheet-polygons-3857/icesheet_polygons.shp
+    archive:
+      format: zip
+      files:
+        - antarctica-icesheet-polygons-3857/icesheet_polygons.cpg
+        - antarctica-icesheet-polygons-3857/icesheet_polygons.dbf
+        - antarctica-icesheet-polygons-3857/icesheet_polygons.prj
+        - antarctica-icesheet-polygons-3857/icesheet_polygons.shp
+        - antarctica-icesheet-polygons-3857/icesheet_polygons.shx
+  icesheet_outlines:
+    type: shp
+    url: https://osmdata.openstreetmap.de/download/antarctica-icesheet-outlines-3857.zip
+    file: antarctica-icesheet-outlines-3857/icesheet_outlines.shp
+    ogropts:
+      - "-explodecollections"
+    archive:
+      format: zip
+      files:
+        - antarctica-icesheet-outlines-3857/icesheet_outlines.cpg
+        - antarctica-icesheet-outlines-3857/icesheet_outlines.dbf
+        - antarctica-icesheet-outlines-3857/icesheet_outlines.prj
+        - antarctica-icesheet-outlines-3857/icesheet_outlines.shp
+        - antarctica-icesheet-outlines-3857/icesheet_outlines.shx
+
+  ne_110m_admin_0_boundary_lines_land:
+    type: shp
+    url: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_admin_0_boundary_lines_land.zip
+    file: ne_110m_admin_0_boundary_lines_land.shp
+    ogropts: &ne_opts
+      - "--config"
+      - "SHAPE_ENCODING"
+      - "WINDOWS-1252"
+      - "-explodecollections"
+      # needs reprojecting
+      - '-t_srs'
+      - 'EPSG:3857'
+    archive:
+      format: zip
+      files:
+        - ne_110m_admin_0_boundary_lines_land.dbf
+        - ne_110m_admin_0_boundary_lines_land.prj
+        - ne_110m_admin_0_boundary_lines_land.shp
+        - ne_110m_admin_0_boundary_lines_land.shx

project.mml

@@ -24,10 +24,6 @@ _parts:
     extent: *world
     srs-name: "900913"
     srs: "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over"
-  extents84: &extents84
-    extent: *world
-    srs-name: "WGS84"
-    srs: "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs"
   osm2pgsql: &osm2pgsql
     type: "postgis"
     dbname: "gis"
@@ -157,8 +153,12 @@ Layer:
     geometry: polygon
     <<: *extents
     Datasource:
-      file: data/antarctica-icesheet-polygons-3857/icesheet_polygons.shp
-      type: shape
+      <<: *osm2pgsql
+      table: |-
+        (SELECT
+            way
+          FROM icesheet_polygons
+        ) AS icesheet_polygons
     properties:
       minzoom: 5
   - id: water-lines-casing
@@ -255,16 +255,24 @@ Layer:
     geometry: polygon
     <<: *extents
     Datasource:
-      file: data/simplified-water-polygons-split-3857/simplified_water_polygons.shp
-      type: shape
+      <<: *osm2pgsql
+      table: |-
+        (SELECT
+            way
+          FROM simplified_water_polygons
+        ) AS ocean_lz
     properties:
       maxzoom: 9
   - id: ocean
     geometry: polygon
     <<: *extents
     Datasource:
-      file: data/water-polygons-split-3857/water_polygons.shp
-      type: shape
+      <<: *osm2pgsql
+      table: |-
+        (SELECT
+            way
+          FROM water_polygons
+        ) AS ocean
     properties:
       minzoom: 10
   - id: landcover-area-symbols
@@ -301,8 +309,13 @@ Layer:
     geometry: linestring
     <<: *extents
     Datasource:
-      file: data/antarctica-icesheet-outlines-3857/icesheet_outlines.shp
-      type: shape
+      <<: *osm2pgsql
+      table: |-
+        (SELECT
+            way,
+            ice_edge
+          FROM icesheet_outlines
+        ) AS icesheet_outlines
     properties:
       minzoom: 5
   - id: marinas-area
@@ -1026,10 +1039,14 @@ Layer:
       minzoom: 11
   - id: necountries
     geometry: linestring
-    <<: *extents84
+    <<: *extents
     Datasource:
-      file: data/ne_110m_admin_0_boundary_lines_land/ne_110m_admin_0_boundary_lines_land.shp
-      type: shape
+      <<: *osm2pgsql
+      table: |-
+        (SELECT
+            way
+          FROM ne_110m_admin_0_boundary_lines_land
+        ) AS necountries
     properties:
       minzoom: 1
       maxzoom: 3

scripts/docker-startup.sh

@@ -50,12 +50,12 @@ EOF
   --style openstreetmap-carto.style \
   --tag-transform-script openstreetmap-carto.lua \
   $OSM2PGSQL_DATAFILE
-  ;;
 
-kosmtik)
   # Downloading needed shapefiles
-  python3 scripts/get-shapefiles.py -n
+  scripts/get-external-data.py
+  ;;
 
+kosmtik)
   # Creating default Kosmtik settings file
   if [ ! -e ".kosmtik-config.yml" ]; then
     cp /tmp/.kosmtik-config.yml .kosmtik-config.yml