Skip to content

Commit

Permalink
Fix for #643 (#808)
Browse files Browse the repository at this point in the history 
* Fix failure to parse geopoint if geoshape question in form
* Re-factored ODK geom to WKT parsing
* Fix logging for xform submissions
  • Loading branch information
@bjohare @ian-ross
bjohare authored and ian-ross committed Oct 10, 2016
1 parent f6b17e2 commit 82c70fc
Show file tree
Hide file tree
Showing 9 changed files with 186 additions and 147 deletions.
39 changes: 2 additions & 37 deletions cadasta/organization/importers/csv.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
import csv
from collections import OrderedDict

from shapely.geometry import LineString, Point, Polygon
from shapely.wkt import dumps

from django.db import transaction
from django.utils.translation import ugettext as _
from party.models import Party, TenureRelationship, TenureRelationshipType
from spatial.models import SpatialUnit
from xforms.utils import odk_geom_to_wkt

from . import base, exceptions

Expand Down Expand Up @@ -104,16 +102,14 @@ def import_data(self, config_dict, **kwargs):
raise ValueError(
_("Invalid geometry type")
)
geometry = self._format_geometry(
coords, geom_type=geometry_type)
geometry = odk_geom_to_wkt(coords)
try:
tenure_type = row[
csv_headers.index(TENURE_TYPE)]
except ValueError as e:
raise ValueError(
_("No 'tenure_type' column found")
)
# get the geom here
content_types['party.party'] = {
'project': self.project,
'name': party_name,
Expand Down Expand Up @@ -149,34 +145,3 @@ def import_data(self, config_dict, **kwargs):
except Exception as e:
raise exceptions.DataImportError(
line_num=reader.line_num, error=e)

def _format_geometry(self, coords, geom_type=None):
if coords == '':
return None
# if '\n' in coords:
# coords = coords.replace('\n', '')
coords = coords.split(';')
if (coords[-1] == ''):
coords.pop()
# fixes bug in geoshape:
# Geoshape copies the second point, not the first.
if geom_type == 'geoshape':
coords.pop()
coords.append(coords[0])

if len(coords) > 1:
points = []
for coord in coords:
coord = coord.split(' ')
coord = [x for x in coord if x]
latlng = [float(coord[1]),
float(coord[0])]
points.append(tuple(latlng))
if (coords[0] != coords[-1] or len(coords) == 2):
return dumps(LineString(points))
else:
return dumps(Polygon(points))
else:
latlng = coords[0].split(' ')
latlng = [x for x in latlng if x]
return dumps(Point(float(latlng[1]), float(latlng[0])))
2 changes: 1 addition & 1 deletion cadasta/organization/tests/files/test_geotrace.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name_mouza,j_l,tenure_type,name_of_HH,name_father_hus,present_add,village_name,M
chanderhowara,90,FH,সুরুজ্জামান,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৮০.০০ অারঙ্গহাটি ৩৬০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,yes,no,yes,no,4,5,yes,no,কিছু বেখলে অাছে জোর করে দখল করে ভোগ করে অন্য জমির মালক।,yes,local_justice,,3913647224045,,,,,,,,2016-02-08T10:43:19.861+06,2016-02-09T09:03:56.526+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:c2b46f0e-dd3c-4c0b-858e-edd582047f6c,5349241,c2b46f0e-dd3c-4c0b-858e-edd582047f6c,2016-02-09T11:30:07,1,,-1,,,201602060307,80437,arifuttaran
chanderhowara,90,FH,অাব্দুল বারেক মন্ডল,মৃত বঙ্গু মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭৯৮০৮১৯৯৪,farmer,Illiterate,৩০.০০ আরঙ্গহাটি ১২০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,no,yes,no,5,4,no,no,,,,,3913647224033,,,,,,,,2016-02-08T11:13:52.453+06,2016-02-09T13:58:53.495+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,5349244,4df81c07-29d2-4a75-9fe4-e7dcc4b05d16,2016-02-09T11:30:17,2,,-1,,,201602060307,96301,arifuttaran
chanderhowara,90,FH,হানিফ উদ্দিন,মৃত মুগল মন্ডল,অারঙ্গহাটি,পশ্চিম অারঙ্গহাটি,০১৭৯৮০৮১৯৯৪,farmer,litteracy,১৫.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,yes,yes,yes,no,3,4,no,no,,,,,3913647225965,,,,,,,,2016-02-08T11:21:20.859+06,2016-02-09T13:58:16.633+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:405a9d1c-3c54-42ca-929d-ab4f261ce59a,5349247,405a9d1c-3c54-42ca-929d-ab4f261ce59a,2016-02-09T11:30:28,3,,-1,,,201602060307,95816,arifuttaran
chanderhowara,90,FH,মো: অামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,service,educated,১৫.০০ অারঙ্গহাটি ৩০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,no,no,yes,no,2,1,no,no,,,,geotrace,3913647224032,45.56342779158167 -122.67650283873081 0.0 0.0;45.56176327330353 -122.67669159919024 0.0 0.0; 45.56151562182025 -122.67490658909082 0.0 0.0; 45.563479432877415 122.67494414001703 0.0 0.0; 45.56176327330353 -122.67669159919024 0.0 0.0;,24.850375,89.832815,17.3,4.8,,,2016-02-08T11:27:32.805+06,2016-02-08T11:55:08.868+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:6925434e-37d3-4f09-b308-1e6e9ccf1d11,5349250,6925434e-37d3-4f09-b308-1e6e9ccf1d11,2016-02-09T11:30:38,4,,-1,,,201602060307,1656,arifuttaran
chanderhowara,90,FH,মো: অামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,service,educated,১৫.০০ অারঙ্গহাটি ৩০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,yes,no,no,yes,no,2,1,no,no,,,,geotrace,3913647224032,45.56342779158167 -122.67650283873081 0.0 0.0;45.56151562182025 -122.67490658909082 0.0 0.0; 45.563479432877415 122.67494414001703 0.0 0.0;45.56176327330353 -122.67669159919024 0.0 0.0;,24.850375,89.832815,17.3,4.8,,,2016-02-08T11:27:32.805+06,2016-02-08T11:55:08.868+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:6925434e-37d3-4f09-b308-1e6e9ccf1d11,5349250,6925434e-37d3-4f09-b308-1e6e9ccf1d11,2016-02-09T11:30:38,4,,-1,,,201602060307,1656,arifuttaran
chanderhowara,90,FH,সোজা মন্ডল,মৃত শাহালী মন্ডল,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,৭০.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,yes,no,no,no,yes,no,0,1,no,no,,,,,3913647224043,,,,,,,,2016-02-08T11:32:28.205+06,2016-02-09T13:58:42.516+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:b2cdbeba-1ced-4740-9605-9430d1ed7ead,5349251,b2cdbeba-1ced-4740-9605-9430d1ed7ead,2016-02-09T11:30:40,5,,-1,,,201602060307,95174,arifuttaran
chanderhowara,90,FH,মিজানুর রহমান,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,15,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,3,2,no,no,,,,,3913647224044,,24.8503433333,89.83295,42.3,7.1,,,2016-02-08T11:37:42.708+06,2016-02-08T11:47:20.509+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:d16a03fa-5009-490e-bc4f-c3812e704b46,5349252,d16a03fa-5009-490e-bc4f-c3812e704b46,2016-02-09T11:30:42,6,,-1,,,201602060307,578,arifuttaran
chanderhowara,90,FH, জামিনুর ইসলাম,সুরুজ্জামান,মন্ডল বাড়ি,চান্দের হাওড়া,০১৭২৮১০১১৩৩,farmer,Illiterate,১৫.০০,১৫.০০,others,inheritance,inheritance,did_not_get_from_tafshil,inheritance,generations,inheritance,no,yes,no,no,yes,no,2,2,no,no,,no,,,3913647224031,,24.8504283333,89.8329316667,13.2,4.9,,,2016-02-08T11:47:29.749+06,2016-02-08T11:51:46.250+06,2016-02-08,355335064140032,470034501263565,8988034503012635653f,,uuid:73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,5349254,73d91b2a-ff43-4c13-b9ec-9d0f9e68d0cb,2016-02-09T11:30:50,7,,-1,,,201602060307,257,arifuttaran
Expand Down
63 changes: 10 additions & 53 deletions cadasta/xforms/mixins/model_helper.py
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,19 @@
from shapely.geometry import LineString, Point, Polygon
from shapely.wkt import dumps

from django.core.exceptions import ValidationError
from django.core.files.storage import get_storage_class
from django.db import transaction
from django.utils.translation import ugettext as _
from jsonattrs.models import Attribute, AttributeType
from party.models import Party, TenureRelationship, TenureRelationshipType
from pyxform.xform2json import XFormToDict
from questionnaires.models import Questionnaire, Question
from questionnaires.models import Questionnaire
from resources.models import Resource
from spatial.models import SpatialUnit
from xforms.exceptions import InvalidXMLSubmission
from xforms.models import XFormSubmission
from xforms.utils import odk_geom_to_wkt


class ModelHelper():
"""
todo:
Update storage after upgrading to latest django-buckets
"""

def __init__(self, *arg):
self.arg = arg

Expand Down Expand Up @@ -87,22 +80,16 @@ def create_spatial_unit(self, data, project, questionnaire, party=None):
for group in location_group:
if 'location_geotrace' in group.keys():
location_geometry = group['location_geotrace']
geoshape = False
elif 'location_geoshape' in group.keys():
location_geometry = group['location_geoshape']
geoshape = True
else:
location_geometry = group['location_geometry']
geoshape = Question.objects.filter(
questionnaire=questionnaire, type='GS').exists()

geom = odk_geom_to_wkt(location_geometry)
location = SpatialUnit.objects.create(
project=project,
type=group['location_type'],
geometry=self._format_geometry(
location_geometry,
geoshape
),
geometry=geom,
attributes=self._get_attributes(group, 'location')
)

Expand Down Expand Up @@ -141,7 +128,7 @@ def create_tenure_relationship(self, data, party, location, project):
attributes=self._get_attributes(
tenure_group[t],
'tenure_relationship')
)
)
tenure_resources.append(
self._get_resource_names(
tenure_group[t], tenure, 'tenure')
Expand Down Expand Up @@ -191,8 +178,8 @@ def upload_submission_data(self, request):
submission = full_submission[list(full_submission.keys())[0]]

with transaction.atomic():
questionnaire, party, location, tenure = self.create_models(
submission)
(questionnaire, party,
location, tenure) = self.create_models(submission)

party_submission = [submission]
location_submission = [submission]
Expand All @@ -205,7 +192,8 @@ def upload_submission_data(self, request):

elif 'location_repeat' in submission:
location_submission = self._format_repeat(
submission, ['location'])
submission, ['location']
)
if 'tenure_type' in location_submission[0]:
tenure_submission = location_submission

Expand Down Expand Up @@ -292,37 +280,6 @@ def upload_resource_files(self, request, data):
content_object=None
)

def _format_geometry(self, coords, geoshape=False):
if coords == '':
return ''
if '\n' in coords:
coords = coords.replace('\n', '')
coords = coords.split(';')
if (coords[-1] == ''):
coords.pop()
# fixes bug in geoshape:
# Geoshape copies the second point, not the first.
if geoshape:
coords.pop()
coords.append(coords[0])

if len(coords) > 1:
points = []
for coord in coords:
coord = coord.split(' ')
coord = [x for x in coord if x]
latlng = [float(coord[1]),
float(coord[0])]
points.append(tuple(latlng))
if (coords[0] != coords[-1] or len(coords) == 2):
return dumps(LineString(points))
else:
return dumps(Polygon(points))
else:
latlng = coords[0].split(' ')
latlng = [x for x in latlng if x]
return dumps(Point(float(latlng[1]), float(latlng[0])))

def _format_repeat(self, data, model_type):
repeat_group = [data]
for model in model_type:
Expand Down Expand Up @@ -350,7 +307,7 @@ def _get_attributes(self, data, model_type):
if Attribute.objects.filter(
name=item,
attr_type=AttributeType.objects.get(
name='select_multiple')).exists():
name='select_multiple')).exists():
answers = data[attr_group][item].split(' ')
attributes[item] = answers
else:
Expand Down
47 changes: 39 additions & 8 deletions cadasta/xforms/tests/files/test_resources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,12 @@
<title />
<party_type>IN</party_type>
<party_name>Peggy Carter</party_name>
<location_geometry>40.6890612 -73.9925067 0.0 0.0;
41.6890612 -73.9925067 0.0 0.0;41.6890612 -72.9925067 0.0 0.0;
40.6890612 -72.9925067 0.0 0.0;40.6890612 -73.9925067 0.0 0.0;
<location_geometry>52.94499198165777 -8.038442619144917 0.0 0.0;
52.943536322452495 -8.038954921066761 0.0 0.0;52.941551455741696
-8.037515245378017 0.0 0.0;
52.94318457288768 -8.034790456295013 0.0 0.0;52.94509582556425
-8.03566887974739 0.0 0.0;
52.943536322452495 -8.038954921066761 0.0 0.0;
</location_geometry>
<location_type>MI</location_type>
<tenure_type>LH</tenure_type>
Expand All @@ -75,11 +78,9 @@
<title />
<party_type>IN</party_type>
<party_name>Buckey Barnes</party_name>
<location_geometry>45.56342779158167 -122.67650283873081 0.0 0.0;
45.56176327330353 -122.67669159919024 0.0 0.0;
45.56151562182025 -122.67490658909082 0.0 0.0;
45.563479432877415 -122.67494414001703 0.0 0.0;
45.56176327330353 -122.67669159919024 0.0 0.0
<location_geometry>52.94144813 -8.03473703 0.0 0.0;52.94134545
-8.03589956 0.0 0.0;52.94124167 -8.03624506 0.0 0.0;52.94129824
-8.03570223 0.0 0.0;
</location_geometry>
<location_type>MI</location_type>
<tenure_type>LH</tenure_type>
Expand Down Expand Up @@ -141,6 +142,35 @@
</meta>
</t_questionnaire_geotype_select>'''.strip()


GEOTRACE_AS_POLY = '''<?xml version=\'1.0\' ?>
<t_questionnaire_geotype_select
id="t_questionnaire_geotype_select" version="20160727122111">
<start>2016-07-07T16:38:20.310-04</start>
<end>2016-07-07T16:39:23.673-04</end>
<today>2016-07-07</today>
<deviceid>00:bb:3a:44:d0:fb</deviceid>
<title />
<party_type>IN</party_type>
<party_name>Natashia Romanoff</party_name>
<location_choice>geotrace</location_choice>
<location_geotrace>
52.9414478 -8.034659 0.0 0.0;
52.94134675 -8.0354197 0.0 0.0;
52.94129841 -8.03517551 0.0 0.0;
52.94142406 -8.03487897 0.0 0.0;
52.9414478 -8.034659 0.0 0.0;
</location_geotrace>
<location_type>MI</location_type>
<tenure_type>LH</tenure_type>
<location_attributes>
<name>geotrace_poly</name>
</location_attributes>
<meta>
<instanceID>uuid:b3f225d3-0fac-4a0b-80c7-60e6db4cc0ad</instanceID>
</meta>
</t_questionnaire_geotype_select>'''.strip()

GEOTYPE_NEITHER = '''<?xml version=\'1.0\' ?>
<t_questionnaire_geotype_select
id="t_questionnaire_geotype_select" version="20160727122111">
Expand Down Expand Up @@ -593,6 +623,7 @@
'submission': STANDARD,
'submission_line': LINE,
'submission_poly': POLY,
'submission_geotrace_as_poly': GEOTRACE_AS_POLY,
'submission_missing_semi': MISSING_SEMI,
'submission_geotype_select': GEOTYPE_SELECT,
'submission_geotype_neither': GEOTYPE_NEITHER,
Expand Down
41 changes: 0 additions & 41 deletions cadasta/xforms/tests/test_model_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -621,47 +621,6 @@ def test_create_resource(self):
self, data, self.user, self.project, content_object='ardvark')
assert Resource.objects.count() == 2

# def test_upload_submission_data(self):
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# covered by the view tests
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# def test_upload_resource_files(self):
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# covered by the view tests
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~

def test_format_geometry(self):
point = '40.6890612 -73.9925067 0.0 0.0;'
geometry = mh._format_geometry(self, point, False)
assert 'POINT' in geometry

point_minus_semi = '340.6890612 -373.9925067 0.0 0.0'
geometry = mh._format_geometry(self, point_minus_semi, False)
assert 'POINT' in geometry

polygon = ('40.6890612 -73.9925067 0.0 0.0;'
'41.6890612 -73.9925067 0.0 0.0;'
'41.6890612 -72.9925067 0.0 0.0;'
'40.6890612 -72.9925067 0.0 0.0;'
'40.6890612 -73.9925067 0.0 0.0;')
geometry = mh._format_geometry(self, polygon, False)
assert 'POLYGON' in geometry

line = ('45.56342779158167 -122.67650283873081 0.0 0.0;'
'45.56176327330353 -122.67669159919024 0.0 0.0;'
'45.56151562182025 -122.67490658909082 0.0 0.0;')
geometry = mh._format_geometry(self, line, False)
assert 'LINESTRING' in geometry

geoshape = ('45.56342779158167 -122.67650283873081 0.0 0.0;'
'45.56176327330353 -122.67669159919024 0.0 0.0;'
'45.56151562182025 -122.67490658909082 0.0 0.0;'
'45.563479432877415 -122.67494414001703 0.0 0.0;'
'45.56176327330353 -122.67669159919024 0.0 0.0')
geometry = mh._format_geometry(self, geoshape, True)
assert 'POLYGON' in geometry

def test_format_repeat(self):
data = {
'party_type': 'Not repeating',
Expand Down
Loading

0 comments on commit 82c70fc

Please sign in to comment.