From c255821276a531692307f3c2cfe4252f6f6c2cad Mon Sep 17 00:00:00 2001 From: bohare Date: Fri, 12 May 2017 16:06:16 +0100 Subject: [PATCH] Fix for #1350: Add support for GPX tracks with multiple segments --- cadasta/core/static/js/map_utils.js | 31 +- cadasta/resources/models.py | 48 +- cadasta/resources/processors/gpx.py | 67 +- .../tests/files/invalid_xml_version.gpx | 113 ++ cadasta/resources/tests/files/routes.gpx | 2 +- .../resources/tests/files/routes_tracks.gpx | 59 + cadasta/resources/tests/files/track_seg.gpx | 1485 +++++++++++++++++ cadasta/resources/tests/files/tracks.gpx | 2 +- cadasta/resources/tests/test_gpx_processor.py | 25 +- cadasta/resources/tests/test_models.py | 39 +- cadasta/resources/utils/io.py | 1 + requirements/common.txt | 1 + 12 files changed, 1810 insertions(+), 63 deletions(-) create mode 100644 cadasta/resources/tests/files/invalid_xml_version.gpx create mode 100644 cadasta/resources/tests/files/routes_tracks.gpx create mode 100644 cadasta/resources/tests/files/track_seg.gpx diff --git a/cadasta/core/static/js/map_utils.js b/cadasta/core/static/js/map_utils.js index 867772949..b1a18d911 100644 --- a/cadasta/core/static/js/map_utils.js +++ b/cadasta/core/static/js/map_utils.js @@ -65,7 +65,7 @@ function renderFeatures(map, featuresUrl, options) { if (options.fitBounds === 'locations') { var bounds = markers.getBounds(); if (bounds.isValid()) { - map.fitBounds(bounds); + map.fitBounds(bounds); } } } @@ -96,7 +96,7 @@ function renderFeatures(map, featuresUrl, options) { } else { map.fitBounds([[-45.0, -180.0], [45.0, 180.0]]); } - + var geoJson = L.geoJson(null, { style: { weight: 2 }, onEachFeature: function(feature, layer) { @@ -105,7 +105,7 @@ function renderFeatures(map, featuresUrl, options) { "

Location" + feature.properties.type + "

" + "
" + options.trans['open'] + "" + - "
"); + ""); } } }); @@ -116,7 +116,7 @@ function renderFeatures(map, featuresUrl, options) { if (options.location) { options.location.addTo(map); - map.fitBounds(options.location.getBounds()); + map.fitBounds(options.location.getBounds()); } else if (projectBounds) { map.fitBounds(projectBounds); } @@ -149,23 +149,24 @@ function switch_layer_controls(map, options){ function add_spatial_resources(map, url){ $.ajax(url).done(function(data){ - if (data.length == 0) return; + if (data.count == 0) return; var spatialResources = {}; - $.each(data, function(idx, resource){ + $.each(data.results, function (idx, resource) { var name = resource.name; var layers = {}; - var group = new L.LayerGroup(); - $.each(resource.spatial_resources, function(i, spatial_resource){ - var layer = L.geoJson(spatial_resource.geom).addTo(group); - layers['name'] = spatial_resource.name; - layers['group'] = group; + $.each(resource.spatial_resources, function (i, spatial_resource) { + var group = new L.LayerGroup(); + var layer = L.geoJson(spatial_resource.geom).addTo(group); + layers[spatial_resource.name] = group }); spatialResources[name] = layers; }); - $.each(spatialResources, function(sr){ - var layer = spatialResources[sr]; - map.layerscontrol.addOverlay(layer['group'], layer['name'], sr); - }) + $.each(spatialResources, function (sr) { + var layers = spatialResources[sr]; + $.each(layers, function (layer) { + map.layerscontrol.addOverlay(layers[layer], layer, sr); + }); + }); }); } diff --git a/cadasta/resources/models.py b/cadasta/resources/models.py index 0662836de..c7d2529ae 100644 --- a/cadasta/resources/models.py +++ b/cadasta/resources/models.py @@ -1,4 +1,6 @@ import os +import tempfile + from datetime import datetime import magic @@ -9,7 +11,6 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db.models import GeometryCollectionField -from django.contrib.gis.gdal.error import GDALException from django.contrib.postgres.fields import JSONField from django.db import models from django.dispatch import receiver @@ -179,36 +180,27 @@ def create_thumbnails(instance, created): def create_spatial_resource(sender, instance, created, **kwargs): if created or instance._original_url != instance.file.url: if instance.mime_type in GPX_MIME_TYPES: - io.ensure_dirs() - file_name = instance.file.url.split('/')[-1] - write_path = os.path.join(settings.MEDIA_ROOT, - 'temp', file_name) - file = instance.file.open().read() - with open(write_path, 'wb') as f: - f.write(file) - # need to double check the mime-type here as browser detection - # of gpx mime type is not reliable - mime = magic.Magic(mime=True) - mime_type = str(mime.from_file(write_path), 'utf-8') - - if mime_type in GPX_MIME_TYPES: - try: - processor = GPXProcessor(write_path) + temp = io.ensure_dirs() + with tempfile.NamedTemporaryFile(mode='wb', dir=temp) as f: + f.write(instance.file.open().read()) + f.seek(0) + # need to double check the mime-type here as browser detection + # of gpx mime type is not reliable + mime = magic.Magic(mime=True) + mime_type = str(mime.from_file(f.name), 'utf-8') + if mime_type in GPX_MIME_TYPES: + processor = GPXProcessor(f.name) layers = processor.get_layers() - except GDALException: + for layer in layers.keys(): + if len(layers[layer]) > 0: + SpatialResource.objects.create( + resource=instance, name=layer, + geom=layers[layer]) + else: raise InvalidGPXFile( - _('Invalid GPX file') + _("Invalid GPX mime type: {error}".format( + error=mime_type)) ) - for layer in layers.keys(): - if len(layers[layer]) > 0: - SpatialResource.objects.create( - resource=instance, name=layer, geom=layers[layer]) - else: - os.remove(write_path) - raise InvalidGPXFile( - _('Invalid GPX mime type: {error}'.format( - error=mime_type)) - ) class ContentObject(RandomIDModel): diff --git a/cadasta/resources/processors/gpx.py b/cadasta/resources/processors/gpx.py index fec47af14..3c008ac22 100644 --- a/cadasta/resources/processors/gpx.py +++ b/cadasta/resources/processors/gpx.py @@ -1,19 +1,68 @@ -from django.contrib.gis.gdal import DataSource -from django.contrib.gis.geos import GeometryCollection +import logging -KEEP_LAYERS = ['tracks', 'routes', 'waypoints'] +import gpxpy +from django.contrib.gis.geos import (GeometryCollection, LineString, + MultiLineString, MultiPoint, Point) +from django.utils.translation import ugettext as _ + +from ..exceptions import InvalidGPXFile + +logger = logging.getLogger(__name__) class GPXProcessor: def __init__(self, gpx_file): - self.ds = DataSource(gpx_file) + with open(gpx_file) as f: + try: + self.gpx = gpxpy.parse(f) + except gpxpy.gpx.GPXException as e: + logger.exception(e) + raise InvalidGPXFile(_("Invalid GPX file: %s" % e)) def get_layers(self): layers = {} - for layer in self.ds: - name = layer.name - if name in KEEP_LAYERS: - # geom = self._get_features(layer) - layers[name] = GeometryCollection(layer.get_geoms(geos=True)) + if self.gpx.tracks: + layers['tracks'] = GeometryCollection( + MultiLineString(parse_tracks(self.gpx.tracks)) + ) + if self.gpx.routes: + layers['routes'] = GeometryCollection( + MultiLineString(parse_routes(self.gpx.routes)) + ) + if self.gpx.waypoints: + layers['waypoints'] = GeometryCollection( + MultiPoint(parse_waypoints(self.gpx.waypoints)) + ) + if not layers: + raise InvalidGPXFile( + _("Error parsing GPX file: no geometry found.")) + return layers + + +def parse_segment(segment): + return [Point(p.longitude, p.latitude).coords for p in segment.points] + + +def parse_tracks(tracks): + multiline = [] + for track in tracks: + for segment in track.segments: + points = parse_segment(segment) + if len(points) > 1: + multiline.append(LineString(points)) + return multiline + + +def parse_waypoints(waypoints): + return [Point(point.longitude, point.latitude) for point in waypoints] + + +def parse_routes(routes): + multiline = [] + for route in routes: + points = parse_segment(route) + if len(points) > 1: + multiline.append(LineString(points)) + return multiline diff --git a/cadasta/resources/tests/files/invalid_xml_version.gpx b/cadasta/resources/tests/files/invalid_xml_version.gpx new file mode 100644 index 000000000..29bf24c43 --- /dev/null +++ b/cadasta/resources/tests/files/invalid_xml_version.gpx @@ -0,0 +1,113 @@ + + + + + Garmin International + + + + + 159.297363 + + 001 + Lodging + + + 88.808327 + + 002 + Lodging + + + 89.445007 + + 003 + Crossing + + + 87.765770 + + 004 + Crossing + + + 88.544846 + + 005 + Crossing + + + 90.230118 + + 006 + Crossing + + + 91.034340 + + 007 + Crossing + + + 90.365944 + + 008 + Crossing + + + 91.291801 + + 009 + Crossing + + + 92.031578 + + 010 + Crossing + + + 89.177856 + + 011 + Crossing + + + 88.838951 + + 012 + Crossing + + + 89.070808 + + 013 + Crossing + + + 92.048050 + + 014 + Crossing + + + 88.055138 + + 015 + Crossing + + + 89.512466 + + 016 + Crossing + + diff --git a/cadasta/resources/tests/files/routes.gpx b/cadasta/resources/tests/files/routes.gpx index 7ffbff9e2..15f91a2b0 100644 --- a/cadasta/resources/tests/files/routes.gpx +++ b/cadasta/resources/tests/files/routes.gpx @@ -1,4 +1,4 @@ - + + + + + Garmin International + + + + + TEST + + 05973230688404 + + + Lough Derg + + + 06061460754125 + + + 05981810818596 + + + + + + 67.0 + + 3.0 + + + 56.0 + + 3.0 + + 1.5299999713897705 + + + + + + 59.0 + + 4.0 + + 0.47999998927116394 + + + + + diff --git a/cadasta/resources/tests/files/track_seg.gpx b/cadasta/resources/tests/files/track_seg.gpx new file mode 100644 index 000000000..9bd4c359b --- /dev/null +++ b/cadasta/resources/tests/files/track_seg.gpx @@ -0,0 +1,1485 @@ + + + + + + 67.0 + + 3.0 + + + 56.0 + + 3.0 + + 1.5299999713897705 + + + + + + 59.0 + + 4.0 + + 0.47999998927116394 + + + + + + 59.0 + + 3.0 + + + 58.0 + + 4.0 + + 2.3499999046325684 + + + + 55.0 + + 3.0 + + 1.1100000143051147 + + + + 52.0 + + 4.0 + + 1.2699999809265137 + + + + 54.0 + + 3.0 + + 1.440000057220459 + + + + 54.0 + + 5.0 + + 2.069999933242798 + + + + 51.0 + + 5.0 + + 0.5400000214576721 + + + + 53.0 + + 7.0 + + 1.600000023841858 + + + + 57.0 + + 4.0 + + 1.600000023841858 + + + + 55.0 + + 9.0 + + 1.159999966621399 + + + + 53.0 + + 12.0 + + 1.2899999618530273 + + + + 50.0 + + 9.0 + + 1.0299999713897705 + + + + 50.0 + + 12.0 + + 1.440000057220459 + + + + 52.0 + + 11.0 + + 1.5399999618530273 + + + + 50.0 + + 10.0 + + 1.4800000190734863 + + + + 55.0 + + 10.0 + + 1.4800000190734863 + + + + 53.0 + + 10.0 + + 1.7000000476837158 + + + + 49.0 + + 9.0 + + 1.4900000095367432 + + + + 55.0 + + 9.0 + + 1.2400000095367432 + + + + 57.0 + + 10.0 + + 1.7699999809265137 + + + + 60.0 + + 9.0 + + 1.7300000190734863 + + + + 57.0 + + 9.0 + + + 52.0 + + 9.0 + + 1.5399999618530273 + + + + 56.0 + + 9.0 + + 1.059999942779541 + + + + 53.0 + + 9.0 + + 1.3200000524520874 + + + + 56.0 + + 9.0 + + 1.600000023841858 + + + + 60.0 + + 9.0 + + 1.6100000143051147 + + + + 59.0 + + 9.0 + + 1.5800000429153442 + + + + 59.0 + + 9.0 + + 1.4700000286102295 + + + + 59.0 + + 9.0 + + 1.2999999523162842 + + + + 64.0 + + 10.0 + + 1.5 + + + + 61.0 + + 10.0 + + 1.5700000524520874 + + + + 56.0 + + 10.0 + + 1.4900000095367432 + + + + 60.0 + + 11.0 + + 1.5099999904632568 + + + + 65.0 + + 9.0 + + 1.7100000381469727 + + + + 63.0 + + 11.0 + + 1.6799999475479126 + + + + 63.0 + + 10.0 + + 1.5 + + + + 66.0 + + 11.0 + + 1.1399999856948853 + + + + 64.0 + + 3.0 + + 1.2200000286102295 + + + + 65.0 + + 3.0 + + 1.5499999523162842 + + + + 64.0 + + 4.0 + + 1.2400000095367432 + + + + 64.0 + + 3.0 + + 1.3300000429153442 + + + + 64.0 + + 4.0 + + 1.5800000429153442 + + + + 59.0 + + 7.0 + + 1.1100000143051147 + + + + 59.0 + + 9.0 + + + 65.0 + + 6.0 + + 1.1799999475479126 + + + + 63.0 + + 11.0 + + 0.7900000214576721 + + + + 62.0 + + 9.0 + + 1.5099999904632568 + + + + 66.0 + + 10.0 + + 1.5700000524520874 + + + + 68.0 + + 10.0 + + 1.2899999618530273 + + + + 69.0 + + 10.0 + + 1.1299999952316284 + + + + 67.0 + + 10.0 + + 1.2899999618530273 + + + + 73.0 + + 10.0 + + 1.4600000381469727 + + + + 75.0 + + 10.0 + + 1.690000057220459 + + + + 76.0 + + 10.0 + + 1.4700000286102295 + + + + 77.0 + + 9.0 + + 1.809999942779541 + + + + 75.0 + + 11.0 + + 1.7400000095367432 + + + + 79.0 + + 10.0 + + 1.0199999809265137 + + + + 75.0 + + 10.0 + + 0.8799999952316284 + + + + 75.0 + + 11.0 + + 0.949999988079071 + + + + 73.0 + + 9.0 + + 0.8100000023841858 + + + + 74.0 + + 10.0 + + 1.1299999952316284 + + + + 76.0 + + 9.0 + + 1.7400000095367432 + + + + 76.0 + + 9.0 + + 1.590000033378601 + + + + 77.0 + + 3.0 + + 1.559999942779541 + + + + 78.0 + + 3.0 + + 1.600000023841858 + + + + 80.0 + + 3.0 + + 1.350000023841858 + + + + 78.0 + + 3.0 + + 1.7200000286102295 + + + + 78.0 + + 3.0 + + 1.5800000429153442 + + + + 76.0 + + 3.0 + + 1.5099999904632568 + + + + 77.0 + + 3.0 + + 1.7100000381469727 + + + + 78.0 + + 4.0 + + 0.7900000214576721 + + + + 78.0 + + 5.0 + + 1.850000023841858 + + + + 79.0 + + 5.0 + + 1.559999942779541 + + + + 81.0 + + 10.0 + + 1.5099999904632568 + + + + 81.0 + + 9.0 + + 1.3600000143051147 + + + + 77.0 + + 10.0 + + 1.3899999856948853 + + + + 78.0 + + 10.0 + + 1.4800000190734863 + + + + 81.0 + + 9.0 + + 1.8200000524520874 + + + + 82.0 + + 10.0 + + 1.5199999809265137 + + + + 84.0 + + 10.0 + + 1.7300000190734863 + + + + 80.0 + + 10.0 + + 1.690000057220459 + + + + 82.0 + + 10.0 + + 1.9199999570846558 + + + + 81.0 + + 9.0 + + 1.4500000476837158 + + + + 75.0 + + 10.0 + + + 82.0 + + 9.0 + + 1.809999942779541 + + + + 80.0 + + 10.0 + + 1.1799999475479126 + + + + 74.0 + + 4.0 + + 1.5199999809265137 + + + + 71.0 + + 5.0 + + 1.440000057220459 + + + + 75.0 + + 3.0 + + 1.5099999904632568 + + + + 79.0 + + 3.0 + + 1.0800000429153442 + + + + 80.0 + + 3.0 + + 1.2200000286102295 + + + + 77.0 + + 5.0 + + 1.5700000524520874 + + + + 77.0 + + 7.0 + + 1.5399999618530273 + + + + 77.0 + + 5.0 + + 1.1799999475479126 + + + + 77.0 + + 11.0 + + 1.2899999618530273 + + + + 82.0 + + 11.0 + + 0.7799999713897705 + + + + 77.0 + + 13.0 + + 1.100000023841858 + + + + 77.0 + + 11.0 + + 1.1799999475479126 + + + + 79.0 + + 12.0 + + 1.2000000476837158 + + + + 77.0 + + 12.0 + + 1.4800000190734863 + + + + 77.0 + + 11.0 + + 0.6200000047683716 + + + + 81.0 + + 11.0 + + 1.1799999475479126 + + + + 74.0 + + 11.0 + + 1.600000023841858 + + + + 78.0 + + 10.0 + + 1.5299999713897705 + + + + 77.0 + + 10.0 + + 2.0999999046325684 + + + + 77.0 + + 5.0 + + 1.399999976158142 + + + + 74.0 + + 4.0 + + 1.3300000429153442 + + + + 75.0 + + 3.0 + + 1.5700000524520874 + + + + 77.0 + + 3.0 + + 1.2599999904632568 + + + + 75.0 + + 4.0 + + 0.550000011920929 + + + + 76.0 + + 5.0 + + 0.6399999856948853 + + + + 74.0 + + 4.0 + + 1.0700000524520874 + + + + 74.0 + + 4.0 + + 2.690000057220459 + + + + 69.0 + + 5.0 + + 0.699999988079071 + + + + 65.0 + + 7.0 + + 0.6200000047683716 + + + + 66.0 + + 5.0 + + + 66.0 + + 13.0 + + + 65.0 + + 12.0 + + + 66.0 + + 10.0 + + + 66.0 + + 10.0 + + + 66.0 + + 10.0 + + + 66.0 + + 10.0 + + + 66.0 + + 10.0 + + + 66.0 + + 10.0 + + + 65.0 + + 10.0 + + 0.9900000095367432 + + + + 70.0 + + 3.0 + + 1.309999942779541 + + + + 66.0 + + 3.0 + + 1.3899999856948853 + + + + 67.0 + + 4.0 + + 0.9800000190734863 + + + + 66.0 + + 6.0 + + 1.6100000143051147 + + + + 68.0 + + 5.0 + + 0.6800000071525574 + + + + 62.0 + + 4.0 + + 1.590000033378601 + + + + 59.0 + + 4.0 + + 0.5299999713897705 + + + + 58.0 + + 4.0 + + 1.8899999856948853 + + + + 54.0 + + 4.0 + + + 55.0 + + 4.0 + + 0.5299999713897705 + + + + 54.0 + + 5.0 + + 0.3799999952316284 + + + + 51.0 + + 5.0 + + + 51.0 + + 3.0 + + + 52.0 + + 3.0 + + + 52.0 + + 3.0 + + 1.6200000047683716 + + + + 50.0 + + 4.0 + + 1.350000023841858 + + + + 48.0 + + 4.0 + + 1.0399999618530273 + + + + 47.0 + + 5.0 + + 0.75 + + + + 51.0 + + 3.0 + + 1.0800000429153442 + + + + 47.0 + + 3.0 + + 1.7799999713897705 + + + + 46.0 + + 4.0 + + 1.9900000095367432 + + + + 47.0 + + 3.0 + + 0.6200000047683716 + + + + 45.0 + + 5.0 + + 1.559999942779541 + + + + 52.0 + + 3.0 + + 3.069999933242798 + + + + 50.0 + + 4.0 + + 2.9000000953674316 + + + + 51.0 + + 3.0 + + 1.7999999523162842 + + + + 55.0 + + 3.0 + + 0.8299999833106995 + + + + 50.0 + + 3.0 + + 1.6699999570846558 + + + + 48.0 + + 3.0 + + 2.049999952316284 + + + + 47.0 + + 4.0 + + 0.7300000190734863 + + + + 45.0 + + 4.0 + + 1.2200000286102295 + + + + 47.0 + + 3.0 + + 1.4199999570846558 + + + + 44.0 + + 3.0 + + 1.909999966621399 + + + + 51.0 + + 3.0 + + 2.3499999046325684 + + + + 47.0 + + 4.0 + + 2.140000104904175 + + + + 50.0 + + 3.0 + + 1.2200000286102295 + + + + 48.0 + + 3.0 + + 2.0399999618530273 + + + + 50.0 + + 3.0 + + 2.0 + + + + 47.0 + + 3.0 + + 1.149999976158142 + + + + 46.0 + + 3.0 + + 1.2799999713897705 + + + + 45.0 + + 4.0 + + 0.6299999952316284 + + + + 46.0 + + 3.0 + + 1.0399999618530273 + + + + 42.0 + + 3.0 + + 1.159999966621399 + + + + 45.0 + + 3.0 + + 2.0299999713897705 + + + + 46.0 + + 4.0 + + 2.299999952316284 + + + + 50.0 + + 3.0 + + 2.25 + + + + 48.0 + + 3.0 + + 2.309999942779541 + + + + 51.0 + + 3.0 + + 2.259999990463257 + + + + 58.0 + + 4.0 + + 1.8600000143051147 + + + + 53.0 + + 4.0 + + 1.4900000095367432 + + + + 49.0 + + 4.0 + + 1.3300000429153442 + + + + 47.0 + + 4.0 + + 1.0700000524520874 + + + + 50.0 + + 4.0 + + 1.1699999570846558 + + + + 48.0 + + 5.0 + + 1.4600000381469727 + + + + 48.0 + + 4.0 + + 1.8899999856948853 + + + + 49.0 + + 5.0 + + 1.5800000429153442 + + + + 50.0 + + 4.0 + + 1.4500000476837158 + + + + 52.0 + + 3.0 + + 1.309999942779541 + + + + 57.0 + + 4.0 + + 1.7799999713897705 + + + + 54.0 + + 4.0 + + 0.699999988079071 + + + + 57.0 + + 10.0 + + 3.0799999237060547 + + + + 112.0 + + 27.0 + + 0.46000000834465027 + + + + + diff --git a/cadasta/resources/tests/files/tracks.gpx b/cadasta/resources/tests/files/tracks.gpx index 7211f0bcd..a24cf186d 100644 --- a/cadasta/resources/tests/files/tracks.gpx +++ b/cadasta/resources/tests/files/tracks.gpx @@ -1,4 +1,4 @@ - +