Fix for #643

* Fix failure to parse geopoint if geoshape question in form
* Re-factored ODK geom to WKT parsing
* Fix logging for xform submissions

cadasta/organization/importers/csv.py

@@ -1,13 +1,11 @@
 import csv
 from collections import OrderedDict
 
-from shapely.geometry import LineString, Point, Polygon
-from shapely.wkt import dumps
-
 from django.db import transaction
 from django.utils.translation import ugettext as _
 from party.models import Party, TenureRelationship, TenureRelationshipType
 from spatial.models import SpatialUnit
+from xforms.utils import odk_geom_to_wkt
 
 from . import base, exceptions
 
@@ -104,16 +102,14 @@ def import_data(self, config_dict, **kwargs):
                             raise ValueError(
                                 _("Invalid geometry type")
                             )
-                        geometry = self._format_geometry(
-                            coords, geom_type=geometry_type)
+                        geometry = odk_geom_to_wkt(coords)
                         try:
                             tenure_type = row[
                                 csv_headers.index(TENURE_TYPE)]
                         except ValueError as e:
                             raise ValueError(
                                 _("No 'tenure_type' column found")
                             )
-                        # get the geom here
                         content_types['party.party'] = {
                             'project': self.project,
                             'name': party_name,
@@ -149,34 +145,3 @@ def import_data(self, config_dict, **kwargs):
         except Exception as e:
             raise exceptions.DataImportError(
                 line_num=reader.line_num, error=e)
-
-    def _format_geometry(self, coords, geom_type=None):
-        if coords == '':
-            return None
-        # if '\n' in coords:
-        #     coords = coords.replace('\n', '')
-        coords = coords.split(';')
-        if (coords[-1] == ''):
-            coords.pop()
-        # fixes bug in geoshape:
-        # Geoshape copies the second point, not the first.
-        if geom_type == 'geoshape':
-            coords.pop()
-            coords.append(coords[0])
-
-        if len(coords) > 1:
-            points = []
-            for coord in coords:
-                coord = coord.split(' ')
-                coord = [x for x in coord if x]
-                latlng = [float(coord[1]),
-                          float(coord[0])]
-                points.append(tuple(latlng))
-            if (coords[0] != coords[-1] or len(coords) == 2):
-                return dumps(LineString(points))
-            else:
-                return dumps(Polygon(points))
-        else:
-            latlng = coords[0].split(' ')
-            latlng = [x for x in latlng if x]
-            return dumps(Point(float(latlng[1]), float(latlng[0])))

cadasta/organization/tests/files/test_geotrace.csv

@@ -2,7 +2,7 @@ name_mouza,j_l,tenure_type,name_of_HH,name_father_hus,present_add,village_name,M
 chanderhowara,90,FH,সুরুজ্জামান,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৮০.০০ অারঙ্গহাটি ৩৬০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,yes,no,yes,no,4,5,yes,no,কিছু বেখলে অাছে জোর করে দখল করে ভোগ করে অন্য জমির মালক।,yes,local_justice,,3913647224045,,,,,,,,2016-02-08T10:43:19.861+06,2016-02-09T09:03:56.526+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:c2b46f0e-dd3c-4c0b-858e-edd582047f6c,5349241,c2b46f0e-dd3c-4c0b-858e-edd582047f6c,2016-02-09T11:30:07,1,,-1,,,201602060307,80437,arifuttaran
 chanderhowara,90,FH,অাব্দুল বারেক মন্ডল,মৃত বঙ্গু মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭৯৮০৮১৯৯৪,farmer,Illiterate,৩০.০০ আরঙ্গহাটি ১২০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,no,yes,no,5,4,no,no,,,,,3913647224033,,,,,,,,2016-02-08T11:13:52.453+06,2016-02-09T13:58:53.495+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,5349244,4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,2016-02-09T11:30:17,2,,-1,,,201602060307,96301,arifuttaran
 chanderhowara,90,FH,হানিফ উদ্দিন,মৃত মুগল মন্ডল,অারঙ্গহাটি,পশ্চিম অারঙ্গহাটি,০১৭৯৮০৮১৯৯৪,farmer,litteracy,১৫.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,yes,yes,no,3,4,no,no,,,,,3913647225965,,,,,,,,2016-02-08T11:21:20.859+06,2016-02-09T13:58:16.633+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:405a9d1c-3c54-42ca-929d-ab4f261ce59a,5349247,405a9d1c-3c54-42ca-929d-ab4f261ce59a,2016-02-09T11:30:28,3,,-1,,,201602060307,95816,arifuttaran
-chanderhowara,90,FH,মো: অামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,service,educated,১৫.০০ অারঙ্গহাটি ৩০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,no,no,yes,no,2,1,no,no,,,,geotrace,3913647224032,45.56342779158167 -122.67650283873081 0.0 0.0;45.56176327330353 -122.67669159919024 0.0 0.0; 45.56151562182025 -122.67490658909082 0.0 0.0; 45.563479432877415 122.67494414001703 0.0 0.0; 45.56176327330353 -122.67669159919024 0.0 0.0;,24.850375,89.832815,17.3,4.8,,,2016-02-08T11:27:32.805+06,2016-02-08T11:55:08.868+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:6925434e-37d3-4f09-b308-1e6e9ccf1d11,5349250,6925434e-37d3-4f09-b308-1e6e9ccf1d11,2016-02-09T11:30:38,4,,-1,,,201602060307,1656,arifuttaran
+chanderhowara,90,FH,মো: অামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,service,educated,১৫.০০ অারঙ্গহাটি ৩০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,no,no,yes,no,2,1,no,no,,,,geotrace,3913647224032,45.56342779158167 -122.67650283873081 0.0 0.0;45.56151562182025 -122.67490658909082 0.0 0.0; 45.563479432877415 122.67494414001703 0.0 0.0;45.56176327330353 -122.67669159919024 0.0 0.0;,24.850375,89.832815,17.3,4.8,,,2016-02-08T11:27:32.805+06,2016-02-08T11:55:08.868+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:6925434e-37d3-4f09-b308-1e6e9ccf1d11,5349250,6925434e-37d3-4f09-b308-1e6e9ccf1d11,2016-02-09T11:30:38,4,,-1,,,201602060307,1656,arifuttaran
 chanderhowara,90,FH,সোজা মন্ডল,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,৭০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,no,no,yes,no,0,1,no,no,,,,,3913647224043,,,,,,,,2016-02-08T11:32:28.205+06,2016-02-09T13:58:42.516+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:b2cdbeba-1ced-4740-9605-9430d1ed7ead,5349251,b2cdbeba-1ced-4740-9605-9430d1ed7ead,2016-02-09T11:30:40,5,,-1,,,201602060307,95174,arifuttaran
 chanderhowara,90,FH,মিজানুর রহমান,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,15,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,3,2,no,no,,,,,3913647224044,,24.8503433333,89.83295,42.3,7.1,,,2016-02-08T11:37:42.708+06,2016-02-08T11:47:20.509+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:d16a03fa-5009-490e-bc4f-c3812e704b46,5349252,d16a03fa-5009-490e-bc4f-c3812e704b46,2016-02-09T11:30:42,6,,-1,,,201602060307,578,arifuttaran
 chanderhowara,90,FH, জামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৫.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,2,2,no,no,,no,,,3913647224031,,24.8504283333,89.8329316667,13.2,4.9,,,2016-02-08T11:47:29.749+06,2016-02-08T11:51:46.250+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,5349254,73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,2016-02-09T11:30:50,7,,-1,,,201602060307,257,arifuttaran

cadasta/xforms/mixins/model_helper.py

@@ -1,26 +1,19 @@
-from shapely.geometry import LineString, Point, Polygon
-from shapely.wkt import dumps
-
 from django.core.exceptions import ValidationError
 from django.core.files.storage import get_storage_class
 from django.db import transaction
 from django.utils.translation import ugettext as _
 from jsonattrs.models import Attribute, AttributeType
 from party.models import Party, TenureRelationship, TenureRelationshipType
 from pyxform.xform2json import XFormToDict
-from questionnaires.models import Questionnaire, Question
+from questionnaires.models import Questionnaire
 from resources.models import Resource
 from spatial.models import SpatialUnit
 from xforms.exceptions import InvalidXMLSubmission
 from xforms.models import XFormSubmission
+from xforms.utils import odk_geom_to_wkt
 
 
 class ModelHelper():
-    """
-    todo:
-    Update storage after upgrading to latest django-buckets
-    """
-
     def __init__(self, *arg):
         self.arg = arg
 
@@ -87,22 +80,16 @@ def create_spatial_unit(self, data, project, questionnaire, party=None):
             for group in location_group:
                 if 'location_geotrace' in group.keys():
                     location_geometry = group['location_geotrace']
-                    geoshape = False
                 elif 'location_geoshape' in group.keys():
                     location_geometry = group['location_geoshape']
-                    geoshape = True
                 else:
                     location_geometry = group['location_geometry']
-                    geoshape = Question.objects.filter(
-                        questionnaire=questionnaire, type='GS').exists()
 
+                geom = odk_geom_to_wkt(location_geometry)
                 location = SpatialUnit.objects.create(
                     project=project,
                     type=group['location_type'],
-                    geometry=self._format_geometry(
-                        location_geometry,
-                        geoshape
-                    ),
+                    geometry=geom,
                     attributes=self._get_attributes(group, 'location')
                 )
 
@@ -141,7 +128,7 @@ def create_tenure_relationship(self, data, party, location, project):
                         attributes=self._get_attributes(
                             tenure_group[t],
                             'tenure_relationship')
-                        )
+                    )
                     tenure_resources.append(
                         self._get_resource_names(
                             tenure_group[t], tenure, 'tenure')
@@ -191,8 +178,8 @@ def upload_submission_data(self, request):
         submission = full_submission[list(full_submission.keys())[0]]
 
         with transaction.atomic():
-            questionnaire, party, location, tenure = self.create_models(
-                                                                    submission)
+            (questionnaire, party,
+                location, tenure) = self.create_models(submission)
 
             party_submission = [submission]
             location_submission = [submission]
@@ -205,7 +192,8 @@ def upload_submission_data(self, request):
 
             elif 'location_repeat' in submission:
                 location_submission = self._format_repeat(
-                                                    submission, ['location'])
+                    submission, ['location']
+                )
                 if 'tenure_type' in location_submission[0]:
                     tenure_submission = location_submission
 
@@ -292,37 +280,6 @@ def upload_resource_files(self, request, data):
                                      content_object=None
                                      )
 
-    def _format_geometry(self, coords, geoshape=False):
-        if coords == '':
-            return ''
-        if '\n' in coords:
-            coords = coords.replace('\n', '')
-        coords = coords.split(';')
-        if (coords[-1] == ''):
-            coords.pop()
-        # fixes bug in geoshape:
-        # Geoshape copies the second point, not the first.
-        if geoshape:
-            coords.pop()
-            coords.append(coords[0])
-
-        if len(coords) > 1:
-            points = []
-            for coord in coords:
-                coord = coord.split(' ')
-                coord = [x for x in coord if x]
-                latlng = [float(coord[1]),
-                          float(coord[0])]
-                points.append(tuple(latlng))
-            if (coords[0] != coords[-1] or len(coords) == 2):
-                return dumps(LineString(points))
-            else:
-                return dumps(Polygon(points))
-        else:
-            latlng = coords[0].split(' ')
-            latlng = [x for x in latlng if x]
-            return dumps(Point(float(latlng[1]), float(latlng[0])))
-
     def _format_repeat(self, data, model_type):
         repeat_group = [data]
         for model in model_type:
@@ -350,7 +307,7 @@ def _get_attributes(self, data, model_type):
                     if Attribute.objects.filter(
                         name=item,
                         attr_type=AttributeType.objects.get(
-                                    name='select_multiple')).exists():
+                            name='select_multiple')).exists():
                         answers = data[attr_group][item].split(' ')
                         attributes[item] = answers
                     else:

cadasta/xforms/tests/files/test_resources.py

@@ -50,9 +50,12 @@
         <title />
         <party_type>IN</party_type>
         <party_name>Peggy Carter</party_name>
-        <location_geometry>40.6890612 -73.9925067 0.0 0.0;
-        41.6890612 -73.9925067 0.0 0.0;41.6890612 -72.9925067 0.0 0.0;
-        40.6890612 -72.9925067 0.0 0.0;40.6890612 -73.9925067 0.0 0.0;
+        <location_geometry>52.94499198165777 -8.038442619144917 0.0 0.0;
+        52.943536322452495 -8.038954921066761 0.0 0.0;52.941551455741696
+        -8.037515245378017 0.0 0.0;
+        52.94318457288768 -8.034790456295013 0.0 0.0;52.94509582556425
+        -8.03566887974739 0.0 0.0;
+        52.943536322452495 -8.038954921066761 0.0 0.0;
         </location_geometry>
         <location_type>MI</location_type>
         <tenure_type>LH</tenure_type>
@@ -75,11 +78,9 @@
         <title />
         <party_type>IN</party_type>
         <party_name>Buckey Barnes</party_name>
-        <location_geometry>45.56342779158167 -122.67650283873081 0.0 0.0;
-        45.56176327330353 -122.67669159919024 0.0 0.0;
-        45.56151562182025 -122.67490658909082 0.0 0.0;
-        45.563479432877415 -122.67494414001703 0.0 0.0;
-        45.56176327330353 -122.67669159919024 0.0 0.0
+        <location_geometry>52.94144813 -8.03473703 0.0 0.0;52.94134545
+        -8.03589956 0.0 0.0;52.94124167 -8.03624506 0.0 0.0;52.94129824
+        -8.03570223 0.0 0.0;
         </location_geometry>
         <location_type>MI</location_type>
         <tenure_type>LH</tenure_type>
@@ -141,6 +142,35 @@
         </meta>
     </t_questionnaire_geotype_select>'''.strip()
 
+
+GEOTRACE_AS_POLY = '''<?xml version=\'1.0\' ?>
+    <t_questionnaire_geotype_select
+        id="t_questionnaire_geotype_select" version="20160727122111">
+        <start>2016-07-07T16:38:20.310-04</start>
+        <end>2016-07-07T16:39:23.673-04</end>
+        <today>2016-07-07</today>
+        <deviceid>00:bb:3a:44:d0:fb</deviceid>
+        <title />
+        <party_type>IN</party_type>
+        <party_name>Natashia Romanoff</party_name>
+        <location_choice>geotrace</location_choice>
+        <location_geotrace>
+        52.9414478 -8.034659 0.0 0.0;
+        52.94134675 -8.0354197 0.0 0.0;
+        52.94129841 -8.03517551 0.0 0.0;
+        52.94142406 -8.03487897 0.0 0.0;
+        52.9414478 -8.034659 0.0 0.0;
+        </location_geotrace>
+        <location_type>MI</location_type>
+        <tenure_type>LH</tenure_type>
+        <location_attributes>
+            <name>geotrace_poly</name>
+        </location_attributes>
+        <meta>
+            <instanceID>uuid:b3f225d3-0fac-4a0b-80c7-60e6db4cc0ad</instanceID>
+        </meta>
+    </t_questionnaire_geotype_select>'''.strip()
+
 GEOTYPE_NEITHER = '''<?xml version=\'1.0\' ?>
     <t_questionnaire_geotype_select
         id="t_questionnaire_geotype_select" version="20160727122111">
@@ -593,6 +623,7 @@
     'submission': STANDARD,
     'submission_line': LINE,
     'submission_poly': POLY,
+    'submission_geotrace_as_poly': GEOTRACE_AS_POLY,
     'submission_missing_semi': MISSING_SEMI,
     'submission_geotype_select': GEOTYPE_SELECT,
     'submission_geotype_neither': GEOTYPE_NEITHER,

cadasta/xforms/tests/test_model_helper.py

@@ -621,47 +621,6 @@ def test_create_resource(self):
                 self, data, self.user, self.project, content_object='ardvark')
         assert Resource.objects.count() == 2
 
-    # def test_upload_submission_data(self):
-        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-        # covered by the view tests
-        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    # def test_upload_resource_files(self):
-        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-        # covered by the view tests
-        # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    def test_format_geometry(self):
-        point = '40.6890612 -73.9925067 0.0 0.0;'
-        geometry = mh._format_geometry(self, point, False)
-        assert 'POINT' in geometry
-
-        point_minus_semi = '340.6890612 -373.9925067 0.0 0.0'
-        geometry = mh._format_geometry(self, point_minus_semi, False)
-        assert 'POINT' in geometry
-
-        polygon = ('40.6890612 -73.9925067 0.0 0.0;'
-                   '41.6890612 -73.9925067 0.0 0.0;'
-                   '41.6890612 -72.9925067 0.0 0.0;'
-                   '40.6890612 -72.9925067 0.0 0.0;'
-                   '40.6890612 -73.9925067 0.0 0.0;')
-        geometry = mh._format_geometry(self, polygon, False)
-        assert 'POLYGON' in geometry
-
-        line = ('45.56342779158167 -122.67650283873081 0.0 0.0;'
-                '45.56176327330353 -122.67669159919024 0.0 0.0;'
-                '45.56151562182025 -122.67490658909082 0.0 0.0;')
-        geometry = mh._format_geometry(self, line, False)
-        assert 'LINESTRING' in geometry
-
-        geoshape = ('45.56342779158167 -122.67650283873081 0.0 0.0;'
-                    '45.56176327330353 -122.67669159919024 0.0 0.0;'
-                    '45.56151562182025 -122.67490658909082 0.0 0.0;'
-                    '45.563479432877415 -122.67494414001703 0.0 0.0;'
-                    '45.56176327330353 -122.67669159919024 0.0 0.0')
-        geometry = mh._format_geometry(self, geoshape, True)
-        assert 'POLYGON' in geometry
-
     def test_format_repeat(self):
         data = {
             'party_type': 'Not repeating',