Add SHP download

.travis.yml

@@ -11,9 +11,14 @@ addons:
 
 before_install:
   - export DEBIAN_FRONTEND=noninteractive
+  - sudo add-apt-repository ppa:ubuntugis/ppa -y
   - sudo -E apt-get -yq update &>> ~/apt-get-update.log
   - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install postgresql-9.4-postgis-2.2 squid3
+  - sudo apt-get -yq install libgdal1-dev
+  - gdal-config --version
   - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
+  - export C_INCLUDE_PATH=/usr/include/gdal
+  - export CPLUS_INCLUDE_PATH=/usr/include/gdal
 
 install:
   - pip install tox==2.3.1

cadasta/organization/download/base.py

@@ -0,0 +1,49 @@
+from jsonattrs.models import Schema
+
+
+class Exporter():
+    def __init__(self, project):
+        self.project = project
+        self._schema_attrs = {}
+
+    def get_schema_attrs(self, content_type):
+        content_type_key = '{}.{}'.format(content_type.app_label,
+                                          content_type.model)
+
+        if content_type_key not in self._schema_attrs.keys():
+            selectors = [
+                self.project.organization.id,
+                self.project.id,
+                self.project.current_questionnaire
+            ]
+            schemas = Schema.objects.lookup(
+                content_type=content_type, selectors=selectors
+            )
+
+            attrs = []
+            if schemas:
+                attrs = [
+                    a for s in schemas
+                    for a in s.attributes.all() if not a.omit
+                ]
+            self._schema_attrs[content_type_key] = attrs
+
+        return self._schema_attrs[content_type_key]
+
+    def get_values(self, item, model_attrs, schema_attrs):
+        values = []
+        for attr in model_attrs:
+            if '.' in attr:
+                attr_items = attr.split('.')
+                value = None
+                for a in attr_items:
+                    value = (getattr(item, a)
+                             if not value else getattr(value, a))
+                values.append(value)
+            else:
+                values.append(getattr(item, attr))
+
+        for attr in schema_attrs:
+            values.append(item.attributes.get(attr.name, ''))
+
+        return values

cadasta/organization/download/shape.py

@@ -0,0 +1,115 @@
+import os
+import csv
+from osgeo import ogr, osr
+from zipfile import ZipFile
+from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.template.loader import render_to_string
+
+from .base import Exporter
+
+MIME_TYPE = 'application/zip'
+
+
+class ShapeExporter(Exporter):
+    def write_items(self, filename, queryset, content_type, model_attrs):
+        schema_attrs = self.get_schema_attrs(content_type)
+        fields = list(model_attrs) + [a.name for a in schema_attrs]
+
+        with open(filename, 'w+', newline='') as csvfile:
+            csvwriter = csv.writer(csvfile)
+            csvwriter.writerow(fields)
+
+            for item in queryset:
+                values = self.get_values(item, model_attrs, schema_attrs)
+                csvwriter.writerow(values)
+
+    def write_relationships(self, filename):
+        content_type = ContentType.objects.get(app_label='party',
+                                               model='tenurerelationship')
+        self.write_items(filename,
+                         self.project.tenure_relationships.all(),
+                         content_type,
+                         ('id', 'party_id', 'spatial_unit_id',
+                          'tenure_type.label'))
+
+    def write_parties(self, filename):
+        content_type = ContentType.objects.get(app_label='party',
+                                               model='party')
+        self.write_items(filename,
+                         self.project.parties.all(),
+                         content_type,
+                         ('id', 'name', 'type'))
+
+    def write_features(self, layers, filename):
+        content_type = ContentType.objects.get(app_label='spatial',
+                                               model='spatialunit')
+        model_attrs = ('id', 'type')
+        schema_attrs = self.get_schema_attrs(content_type)
+
+        with open(filename, 'w+', newline='') as csvfile:
+            csvwriter = csv.writer(csvfile)
+            csvwriter.writerow(list(model_attrs) +
+                               [a.name for a in schema_attrs])
+
+            for su in self.project.spatial_units.all():
+                geom = ogr.CreateGeometryFromWkt(su.geometry.wkt)
+                layer_type = geom.GetGeometryType() - 1
+                layer = layers[layer_type]
+
+                feature = ogr.Feature(layer.GetLayerDefn())
+                feature.SetGeometry(ogr.CreateGeometryFromWkt(su.geometry.wkt))
+                feature.SetField('id', su.id)
+                layer.CreateFeature(feature)
+                feature.Destroy()
+
+                values = self.get_values(su, model_attrs, schema_attrs)
+                csvwriter.writerow(values)
+
+    def create_datasource(self, dst_dir, f_name):
+        if not os.path.exists(dst_dir):
+            os.makedirs(dst_dir)
+        path = os.path.join(dst_dir, f_name + '-point.shp')
+        driver = ogr.GetDriverByName('ESRI Shapefile')
+        return driver.CreateDataSource(path)
+
+    def create_shp_layers(self, datasource, f_name):
+        srs = osr.SpatialReference()
+        srs.ImportFromEPSG(4326)
+
+        layers = (
+            datasource.CreateLayer(f_name + '-point', srs, geom_type=1),
+            datasource.CreateLayer(f_name + '-line', srs, geom_type=2),
+            datasource.CreateLayer(f_name + '-polygon', srs, geom_type=3)
+        )
+
+        for layer in layers:
+            field = ogr.FieldDefn('id', ogr.OFTString)
+            layer.CreateField(field)
+
+        return layers
+
+    def make_download(self, f_name):
+        dst_dir = os.path.join(settings.MEDIA_ROOT, 'temp/{}'.format(f_name))
+
+        ds = self.create_datasource(dst_dir, self.project.slug)
+        layers = self.create_shp_layers(ds, self.project.slug)
+
+        self.write_features(layers, os.path.join(dst_dir, 'locations.csv'))
+        self.write_relationships(os.path.join(dst_dir, 'relationships.csv'))
+        self.write_parties(os.path.join(dst_dir, 'parties.csv'))
+
+        ds.Destroy()
+
+        path = os.path.join(settings.MEDIA_ROOT, 'temp/{}.zip'.format(f_name))
+        readme = render_to_string(
+            'organization/download/shp_readme.txt',
+            {'project_name': self.project.name,
+             'project_slug': self.project.slug}
+        )
+        with ZipFile(path, 'a') as myzip:
+            myzip.writestr('README.txt', readme)
+            for file in os.listdir(dst_dir):
+                myzip.write(os.path.join(dst_dir, file), arcname=file)
+
+        return path, MIME_TYPE

cadasta/organization/download/xls.py

@@ -2,50 +2,23 @@
 from openpyxl import Workbook
 from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
-from jsonattrs.models import Schema
 
+from .base import Exporter
 
-MIME_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
 
+MIME_TYPE = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
 
-class XLSExporter():
-    def __init__(self, project):
-        self.project = project
 
+class XLSExporter(Exporter):
     def write_items(self, worksheet, queryset, content_type, model_attrs):
-        selectors = [
-            self.project.organization.id,
-            self.project.id,
-            self.project.current_questionnaire
-        ]
-        schemas = Schema.objects.lookup(
-            content_type=content_type, selectors=selectors
-        )
-        attrs = []
-        if schemas:
-            attrs = [a for s in schemas
-                     for a in s.attributes.all() if not a.omit]
+        schema_attrs = self.get_schema_attrs(content_type)
 
         # write column labels
-        worksheet.append(model_attrs + [a.name for a in attrs])
+        worksheet.append(model_attrs + [a.name for a in schema_attrs])
 
         # write data
         for i, item in enumerate(queryset):
-            values = []
-            for j, attr in enumerate(model_attrs):
-                if '.' in attr:
-                    attr_items = attr.split('.')
-                    value = None
-                    for a in attr_items:
-                        value = (getattr(item, a)
-                                 if not value else getattr(value, a))
-                    values.append(value)
-                else:
-                    values.append(getattr(item, attr))
-
-            for j, attr in enumerate(attrs):
-                values.append(item.attributes.get(attr.name, ''))
-
+            values = self.get_values(item, model_attrs, schema_attrs)
             worksheet.append(values)
 
     def write_locations(self):
@@ -61,7 +34,8 @@ def write_parties(self):
         parties = self.project.parties.all()
         content_type = ContentType.objects.get(app_label='party',
                                                model='party')
-        self.write_items(worksheet, parties, content_type, ['id', 'name'])
+        self.write_items(worksheet, parties, content_type,
+                         ['id', 'name', 'type'])
 
     def write_relationships(self):
         worksheet = self.workbook.create_sheet(title='relationships')

cadasta/organization/forms.py

@@ -20,6 +20,7 @@
 from .fields import ProjectRoleField, PublicPrivateField, ContactsField
 from .download.xls import XLSExporter
 from .download.resources import ResourceExporter
+from .download.shape import ShapeExporter
 
 FORM_CHOICES = ROLE_CHOICES + (('Pb', _('Public User')),)
 QUESTIONNAIRE_TYPES = [
@@ -375,7 +376,8 @@ def save(self):
 
 
 class DownloadForm(forms.Form):
-    CHOICES = (('all', 'All data'), ('xls', 'XLS'), ('res', 'Resources'))
+    CHOICES = (('all', 'All data'), ('xls', 'XLS'), ('shp', 'SHP'),
+               ('res', 'Resources'))
     type = forms.ChoiceField(choices=CHOICES, initial='xls')
 
     def __init__(self, project, user, *args, **kwargs):
@@ -389,7 +391,10 @@ def get_file(self):
         file_name = '{}-{}-{}'.format(self.project.id, self.user.id, t)
         type = self.cleaned_data['type']
 
-        if type == 'xls':
+        if type == 'shp':
+            e = ShapeExporter(self.project)
+            path, mime = e.make_download(file_name + '-shp')
+        elif type == 'xls':
             e = XLSExporter(self.project)
             path, mime = e.make_download(file_name + '-xls')
         elif type == 'res':
@@ -398,11 +403,14 @@ def get_file(self):
         elif type == 'all':
             res_exporter = ResourceExporter(self.project)
             xls_exporter = XLSExporter(self.project)
+            shp_exporter = ShapeExporter(self.project)
             path, mime = res_exporter.make_download(file_name + '-res')
             data_path, _ = xls_exporter.make_download(file_name + '-xls')
+            shp_path, _ = shp_exporter.make_download(file_name + '-shp')
 
             with ZipFile(path, 'a') as myzip:
                 myzip.write(data_path, arcname='data.xlsx')
+                myzip.write(shp_path, arcname='data-shp.zip')
                 myzip.close()
 
         return path, mime