Add support for GPX tracks with multiple nested track segments (#1503)

* Fix for #1350: Add support for GPX tracks with multiple segments

* Remove gpx parse exception logging

* Possible fix for gpxpy logging issues

cadasta/core/static/js/map_utils.js

@@ -65,7 +65,7 @@ function renderFeatures(map, featuresUrl, options) {
         if (options.fitBounds === 'locations') {
           var bounds = markers.getBounds();
           if (bounds.isValid()) {
-            map.fitBounds(bounds);  
+            map.fitBounds(bounds);
           }
         }
       }
@@ -96,7 +96,7 @@ function renderFeatures(map, featuresUrl, options) {
   } else {
     map.fitBounds([[-45.0, -180.0], [45.0, 180.0]]);
   }
-  
+
   var geoJson = L.geoJson(null, {
     style: { weight: 2 },
     onEachFeature: function(feature, layer) {
@@ -105,7 +105,7 @@ function renderFeatures(map, featuresUrl, options) {
                       "<h2><span class=\"entity\">Location</span>" +
                       feature.properties.type + "</h2></div>" +
                       "<div class=\"btn-wrap\"><a href='" + feature.properties.url + "' class=\"btn btn-primary btn-sm btn-block\">" + options.trans['open'] + "</a>"  +
-                      "</div>");  
+                      "</div>");
       }
     }
   });
@@ -116,7 +116,7 @@ function renderFeatures(map, featuresUrl, options) {
 
   if (options.location) {
     options.location.addTo(map);
-    map.fitBounds(options.location.getBounds());  
+    map.fitBounds(options.location.getBounds());
   } else if (projectBounds) {
     map.fitBounds(projectBounds);
   }
@@ -149,23 +149,24 @@ function switch_layer_controls(map, options){
 
 function add_spatial_resources(map, url){
   $.ajax(url).done(function(data){
-    if (data.length == 0) return;
+    if (data.count == 0) return;
     var spatialResources = {};
-    $.each(data, function(idx, resource){
+    $.each(data.results, function (idx, resource) {
       var name = resource.name;
       var layers = {};
-      var group = new L.LayerGroup();
-      $.each(resource.spatial_resources, function(i, spatial_resource){
-        var layer = L.geoJson(spatial_resource.geom).addTo(group);
-        layers['name'] = spatial_resource.name;
-        layers['group'] = group;
+      $.each(resource.spatial_resources, function (i, spatial_resource) {
+          var group = new L.LayerGroup();
+          var layer = L.geoJson(spatial_resource.geom).addTo(group);
+          layers[spatial_resource.name] = group
       });
       spatialResources[name] = layers;
     });
-    $.each(spatialResources, function(sr){
-      var layer = spatialResources[sr];
-      map.layerscontrol.addOverlay(layer['group'], layer['name'], sr);
-    })
+    $.each(spatialResources, function (sr) {
+      var layers = spatialResources[sr];
+      $.each(layers, function (layer) {
+          map.layerscontrol.addOverlay(layers[layer], layer, sr);
+      });
+    });
   });
 }
 

cadasta/resources/models.py

@@ -1,4 +1,6 @@
 import os
+import tempfile
+
 from datetime import datetime
 
 import magic
@@ -9,7 +11,6 @@
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.contrib.gis.db.models import GeometryCollectionField
-from django.contrib.gis.gdal.error import GDALException
 from django.contrib.postgres.fields import JSONField
 from django.db import models
 from django.dispatch import receiver
@@ -179,36 +180,27 @@ def create_thumbnails(instance, created):
 def create_spatial_resource(sender, instance, created, **kwargs):
     if created or instance._original_url != instance.file.url:
         if instance.mime_type in GPX_MIME_TYPES:
-            io.ensure_dirs()
-            file_name = instance.file.url.split('/')[-1]
-            write_path = os.path.join(settings.MEDIA_ROOT,
-                                      'temp', file_name)
-            file = instance.file.open().read()
-            with open(write_path, 'wb') as f:
-                f.write(file)
-            # need to double check the mime-type here as browser detection
-            # of gpx mime type is not reliable
-            mime = magic.Magic(mime=True)
-            mime_type = str(mime.from_file(write_path), 'utf-8')
-
-            if mime_type in GPX_MIME_TYPES:
-                try:
-                    processor = GPXProcessor(write_path)
+            temp = io.ensure_dirs()
+            with tempfile.NamedTemporaryFile(mode='wb', dir=temp) as f:
+                f.write(instance.file.open().read())
+                f.seek(0)
+                # need to double check the mime-type here as browser detection
+                # of gpx mime type is not reliable
+                mime = magic.Magic(mime=True)
+                mime_type = str(mime.from_file(f.name), 'utf-8')
+                if mime_type in GPX_MIME_TYPES:
+                    processor = GPXProcessor(f.name)
                     layers = processor.get_layers()
-                except GDALException:
+                    for layer in layers.keys():
+                        if len(layers[layer]) > 0:
+                            SpatialResource.objects.create(
+                                resource=instance, name=layer,
+                                geom=layers[layer])
+                else:
                     raise InvalidGPXFile(
-                        _('Invalid GPX file')
+                        _("Invalid GPX mime type: {error}".format(
+                            error=mime_type))
                     )
-                for layer in layers.keys():
-                    if len(layers[layer]) > 0:
-                        SpatialResource.objects.create(
-                            resource=instance, name=layer, geom=layers[layer])
-            else:
-                os.remove(write_path)
-                raise InvalidGPXFile(
-                    _('Invalid GPX mime type: {error}'.format(
-                        error=mime_type))
-                )
 
 
 class ContentObject(RandomIDModel):

cadasta/resources/processors/gpx.py

@@ -1,19 +1,76 @@
-from django.contrib.gis.gdal import DataSource
-from django.contrib.gis.geos import GeometryCollection
+from django.contrib.gis.geos import (GeometryCollection, LineString,
+                                     MultiLineString, MultiPoint, Point)
+from django.utils.translation import ugettext as _
+from gpxpy import gpx, parser
 
-KEEP_LAYERS = ['tracks', 'routes', 'waypoints']
+from ..exceptions import InvalidGPXFile
+
+
+class GPXParser(parser.GPXParser):
+
+    def parse(self, version=None):
+        try:
+            self.xml_parser = parser.LXMLParser(self.xml)
+            self.__parse_dom(version)
+            return self.gpx
+        except Exception as e:
+            raise gpx.GPXXMLSyntaxException(
+                'Error parsing XML: %s' % str(e), e)
 
 
 class GPXProcessor:
 
     def __init__(self, gpx_file):
-        self.ds = DataSource(gpx_file)
+        with open(gpx_file, 'r') as f:
+            try:
+                parser = GPXParser(f)
+                self.gpx = parser.parse(f)
+            except gpx.GPXException as e:
+                raise InvalidGPXFile(_("Invalid GPX file: %s" % str(e)))
 
     def get_layers(self):
         layers = {}
-        for layer in self.ds:
-            name = layer.name
-            if name in KEEP_LAYERS:
-                # geom = self._get_features(layer)
-                layers[name] = GeometryCollection(layer.get_geoms(geos=True))
+        if self.gpx.tracks:
+            layers['tracks'] = GeometryCollection(
+                MultiLineString(parse_tracks(self.gpx.tracks))
+            )
+        if self.gpx.routes:
+            layers['routes'] = GeometryCollection(
+                MultiLineString(parse_routes(self.gpx.routes))
+            )
+        if self.gpx.waypoints:
+            layers['waypoints'] = GeometryCollection(
+                MultiPoint(parse_waypoints(self.gpx.waypoints))
+            )
+        if not layers:
+            raise InvalidGPXFile(
+                _("Error parsing GPX file: no geometry found."))
+
         return layers
+
+
+def parse_segment(segment):
+    return [Point(p.longitude, p.latitude).coords for p in segment.points]
+
+
+def parse_tracks(tracks):
+    multiline = []
+    for track in tracks:
+        for segment in track.segments:
+            points = parse_segment(segment)
+            if len(points) > 1:
+                multiline.append(LineString(points))
+    return multiline
+
+
+def parse_waypoints(waypoints):
+    return [Point(point.longitude, point.latitude) for point in waypoints]
+
+
+def parse_routes(routes):
+    multiline = []
+    for route in routes:
+        points = parse_segment(route)
+        if len(points) > 1:
+            multiline.append(LineString(points))
+    return multiline

cadasta/resources/tests/files/invalid_xml_version.gpx

@@ -0,0 +1,113 @@
+<?xml version="2.0" encoding="utf-8" standalone="no" ?><!-- invalid version only 1.0 or 1.1 supported -->
+<gpx
+    xmlns="http://www.topografix.com/GPX/1/1"
+    xmlns:gpxx="http://www.garmin.com/xmlschemas/WaypointExtension/v1"
+    xmlns:gpxtrx="http://www.garmin.com/xmlschemas/GpxExtensions/v3"
+    xmlns:gpxtpx="http://www.garmin.com/xmlschemas/TrackPointExtension/v1"
+    creator="Oregon 550"
+    version="1.1"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd http://www.garmin.com/xmlschemas/WaypointExtension/v1 http://www8.garmin.com/xmlschemas/WaypointExtensionv1.xsd http://www.garmin.com/xmlschemas/TrackPointExtension/v1 http://www.garmin.com/xmlschemas/TrackPointExtensionv1.xsd">
+    <metadata>
+        <link href="http://www.garmin.com">
+            <text>Garmin International</text>
+        </link>
+        <time>2016-05-18T09:20:06Z</time>
+    </metadata>
+    <wpt lat="52.941277" lon="-8.034792">
+        <ele>159.297363</ele>
+        <time>2016-05-18T09:20:06Z</time>
+        <name>001</name>
+        <sym>Lodging</sym>
+    </wpt>
+    <wpt lat="52.946419" lon="-8.039001">
+        <ele>88.808327</ele>
+        <time>2016-05-18T10:04:17Z</time>
+        <name>002</name>
+        <sym>Lodging</sym>
+    </wpt>
+    <wpt lat="52.946421" lon="-8.039008">
+        <ele>89.445007</ele>
+        <time>2016-05-18T10:06:58Z</time>
+        <name>003</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946064" lon="-8.038319">
+        <ele>87.765770</ele>
+        <time>2016-05-18T10:08:56Z</time>
+        <name>004</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946153" lon="-8.037743">
+        <ele>88.544846</ele>
+        <time>2016-05-18T10:09:42Z</time>
+        <name>005</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946521" lon="-8.037001">
+        <ele>90.230118</ele>
+        <time>2016-05-18T10:10:45Z</time>
+        <name>006</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946283" lon="-8.036948">
+        <ele>91.034340</ele>
+        <time>2016-05-18T10:11:28Z</time>
+        <name>007</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946500" lon="-8.036520">
+        <ele>90.365944</ele>
+        <time>2016-05-18T10:12:52Z</time>
+        <name>008</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946646" lon="-8.036945">
+        <ele>91.291801</ele>
+        <time>2016-05-18T10:14:18Z</time>
+        <name>009</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946733" lon="-8.037329">
+        <ele>92.031578</ele>
+        <time>2016-05-18T10:14:57Z</time>
+        <name>010</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946578" lon="-8.037304">
+        <ele>89.177856</ele>
+        <time>2016-05-18T10:15:30Z</time>
+        <name>011</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946655" lon="-8.037608">
+        <ele>88.838951</ele>
+        <time>2016-05-18T10:16:14Z</time>
+        <name>012</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946620" lon="-8.037936">
+        <ele>89.070808</ele>
+        <time>2016-05-18T10:16:53Z</time>
+        <name>013</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946616" lon="-8.037954">
+        <ele>92.048050</ele>
+        <time>2016-05-18T10:17:13Z</time>
+        <name>014</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946628" lon="-8.037980">
+        <ele>88.055138</ele>
+        <time>2016-05-18T10:17:40Z</time>
+        <name>015</name>
+        <sym>Crossing</sym>
+    </wpt>
+    <wpt lat="52.946757" lon="-8.037970">
+        <ele>89.512466</ele>
+        <time>2016-05-18T10:18:39Z</time>
+        <name>016</name>
+        <sym>Crossing</sym>
+    </wpt>
+</gpx>

cadasta/resources/tests/files/routes.gpx

@@ -1,4 +1,4 @@
-<?xml version="2.0" encoding="UTF-8" standalone="no" ?>
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
 <gpx
     xmlns="http://www.topografix.com/GPX/1/1"
     xmlns:gpxx="http://www.garmin.com/xmlschemas/WaypointExtension/v1"