Split geoserver.helpers.set_attributes()

geonode/contrib/geosites/models.py

@@ -21,6 +21,7 @@
 from django.db import models
 from django.db.models import signals
 from django.contrib.sites.models import Site
+from django.conf import settings
 
 from geonode.base.models import ResourceBase
 from geonode.layers.models import Layer
@@ -98,13 +99,14 @@ def post_delete_profile(instance, sender, **kwargs):
 
 
 # Django doesn't propagate the signals to the parents so we need to add the listeners on the children
-signals.post_save.connect(post_save_resource, sender=Layer)
-signals.post_save.connect(post_save_resource, sender=Map)
-signals.post_save.connect(post_save_resource, sender=Document)
-signals.post_save.connect(post_save_site, sender=Site)
-signals.post_delete.connect(post_delete_resource, sender=Layer)
-signals.post_delete.connect(post_delete_resource, sender=Map)
-signals.post_delete.connect(post_delete_resource, sender=Document)
-signals.post_delete.connect(post_delete_site, sender=Site)
-signals.post_save.connect(post_save_profile, sender=Profile)
-signals.post_delete.connect(post_delete_profile, sender=Profile)
+if 'geonode.contrib.geosites' in settings.INSTALLED_APPS:
+    signals.post_save.connect(post_save_resource, sender=Layer)
+    signals.post_save.connect(post_save_resource, sender=Map)
+    signals.post_save.connect(post_save_resource, sender=Document)
+    signals.post_save.connect(post_save_site, sender=Site)
+    signals.post_delete.connect(post_delete_resource, sender=Layer)
+    signals.post_delete.connect(post_delete_resource, sender=Map)
+    signals.post_delete.connect(post_delete_resource, sender=Document)
+    signals.post_delete.connect(post_delete_site, sender=Site)
+    signals.post_save.connect(post_save_profile, sender=Profile)
+    signals.post_delete.connect(post_delete_profile, sender=Profile)

geonode/geoserver/helpers.py

@@ -18,58 +18,54 @@
 #
 #########################################################################
 
+from collections import namedtuple, defaultdict
+import datetime
+from decimal import Decimal
+import errno
+from itertools import cycle, izip
 import json
-import sys
-import os
-import urllib
 import logging
+import os
 import re
+import sys
+from threading import local
 import time
-import errno
-import uuid
-import datetime
-from bs4 import BeautifulSoup
-import geoserver
-import httplib2
-
-
+import urllib
 from urlparse import urlparse
 from urlparse import urlsplit
-from threading import local
-from collections import namedtuple
-from itertools import cycle, izip
-from lxml import etree
-import xml.etree.ElementTree as ET
-from decimal import Decimal
-
-from owslib.wcs import WebCoverageService
-from owslib.util import http_post
+import uuid
 
-from django.core.exceptions import ImproperlyConfigured
+from agon_ratings.models import OverallRating
+from bs4 import BeautifulSoup
+from dialogos.models import Comment
+from django.conf import settings
 from django.contrib.contenttypes.models import ContentType
+from django.core.exceptions import ImproperlyConfigured
 from django.db.models.signals import pre_delete
 from django.template.loader import render_to_string
-from django.conf import settings
 from django.utils.translation import ugettext as _
-
-from dialogos.models import Comment
-from agon_ratings.models import OverallRating
-
-from gsimporter import Client
-from owslib.wms import WebMapService
-from geoserver.store import CoverageStore, DataStore, datastore_from_index,\
-    coveragestore_from_index, wmsstore_from_index
-from geoserver.workspace import Workspace
+import geoserver
 from geoserver.catalog import Catalog
-from geoserver.catalog import FailedRequestError, UploadError
 from geoserver.catalog import ConflictingDataError
+from geoserver.catalog import FailedRequestError, UploadError
 from geoserver.resource import FeatureType, Coverage
+from geoserver.store import CoverageStore, DataStore, datastore_from_index, \
+    coveragestore_from_index, wmsstore_from_index
 from geoserver.support import DimensionInfo
+from geoserver.workspace import Workspace
+from gsimporter import Client
+import httplib2
+from lxml import etree
+from owslib.util import http_post
+from owslib.wcs import WebCoverageService
+from owslib.wms import WebMapService
 
 from geonode import GeoNodeException
-from geonode.layers.utils import layer_type, get_files
-from geonode.layers.models import Layer, Attribute, Style
 from geonode.layers.enumerations import LAYER_ATTRIBUTE_NUMERIC_DATA_TYPES
+from geonode.layers.models import Layer, Attribute, Style
+from geonode.layers.utils import layer_type, get_files
+from geonode.utils import set_attributes
+import xml.etree.ElementTree as ET
 
 
 logger = logging.getLogger(__name__)
@@ -467,7 +463,7 @@ def gs_slurp(
             })
 
             # recalculate the layer statistics
-            set_attributes(layer, overwrite=True)
+            set_attributes_from_geoserver(layer, overwrite=True)
 
             # Fix metadata links if the ip has changed
             if layer.link_set.metadata().count() > 0:
@@ -631,7 +627,7 @@ def get_stores(store_type=None):
     return store_list
 
 
-def set_attributes(layer, overwrite=False):
+def set_attributes_from_geoserver(layer, overwrite=False):
     """
     Retrieve layer attribute names & types from Geoserver,
     then store in GeoNode database using Attribute model
@@ -713,73 +709,28 @@ def set_attributes(layer, overwrite=False):
         except Exception:
             attribute_map = []
 
-    # we need 3 more items for description, attribute_label and display_order
-    attribute_map_dict = {
-        'field': 0,
-        'ftype': 1,
-        'description': 2,
-        'label': 3,
-        'display_order': 4,
-    }
-    for attribute in attribute_map:
-        attribute.extend((None, None, 0))
-
-    attributes = layer.attribute_set.all()
-    # Delete existing attributes if they no longer exist in an updated layer
-    for la in attributes:
-        lafound = False
-        for attribute in attribute_map:
-            field, ftype, description, label, display_order = attribute
-            if field == la.attribute:
-                lafound = True
-                # store description and attribute_label in attribute_map
-                attribute[attribute_map_dict['description']] = la.description
-                attribute[attribute_map_dict['label']] = la.attribute_label
-                attribute[attribute_map_dict['display_order']] = la.display_order
-        if overwrite or not lafound:
-            logger.debug(
-                "Going to delete [%s] for [%s]",
-                la.attribute,
-                layer.name.encode('utf-8'))
-            la.delete()
-
+    # Get attribute statistics & package for call to really_set_attributes()
+    attribute_stats = defaultdict(dict)
     # Add new layer attributes if they don't already exist
-    if attribute_map is not None:
-        iter = len(Attribute.objects.filter(layer=layer)) + 1
-        for attribute in attribute_map:
-            field, ftype, description, label, display_order = attribute
-            if field is not None:
-                la, created = Attribute.objects.get_or_create(
-                    layer=layer, attribute=field, attribute_type=ftype,
-                    description=description, attribute_label=label,
-                    display_order=display_order)
-                if created:
-                    if is_layer_attribute_aggregable(
-                            layer.storeType,
-                            field,
-                            ftype):
-                        logger.debug("Generating layer attribute statistics")
-                        result = get_attribute_statistics(layer.name, field)
-                        if result is not None:
-                            la.count = result['Count']
-                            la.min = result['Min']
-                            la.max = result['Max']
-                            la.average = result['Average']
-                            la.median = result['Median']
-                            la.stddev = result['StandardDeviation']
-                            la.sum = result['Sum']
-                            la.unique_values = result['unique_values']
-                            la.last_stats_updated = datetime.datetime.now()
-                    la.visible = ftype.find("gml:") != 0
-                    la.display_order = iter
-                    la.save()
-                    iter += 1
-                    logger.debug(
-                        "Created [%s] attribute for [%s]",
+    for attribute in attribute_map:
+        field, ftype = attribute
+        if field is not None:
+            if Attribute.objects.filter(layer=layer, attribute=field).exists():
+                continue
+            else:
+                if is_layer_attribute_aggregable(
+                        layer.storeType,
                         field,
-                        layer.name.encode('utf-8'))
-    else:
-        logger.debug("No attributes found")
+                        ftype):
+                    logger.debug("Generating layer attribute statistics")
+                    result = get_attribute_statistics(layer.name, field)
+                else:
+                    result = None
+                attribute_stats[layer.name][field] = result
+
+    set_attributes(
+        layer, attribute_map, overwrite=overwrite, attribute_stats=attribute_stats
+    )
 
 
 def set_styles(layer, gs_catalog):
@@ -1180,7 +1131,7 @@ def geoserver_upload(
         box = gs_resource.native_bbox[:4]
         minx, maxx, miny, maxy = [float(a) for a in box]
         if -180 <= minx <= 180 and -180 <= maxx <= 180 and \
-           -90 <= miny <= 90 and -90 <= maxy <= 90:
+           - 90 <= miny <= 90 and -90 <= maxy <= 90:
             logger.info('GeoServer failed to detect the projection for layer '
                         '[%s]. Guessing EPSG:4326', name)
             # If GeoServer couldn't figure out the projection, we just

geonode/geoserver/signals.py

@@ -29,7 +29,7 @@
 from django.conf import settings
 
 from geonode.geoserver.ows import wcs_links, wfs_links, wms_links
-from geonode.geoserver.helpers import cascading_delete, set_attributes
+from geonode.geoserver.helpers import cascading_delete, set_attributes_from_geoserver
 from geonode.geoserver.helpers import set_styles, gs_catalog
 from geonode.geoserver.helpers import ogc_server_settings
 from geonode.geoserver.helpers import geoserver_upload, http_client
@@ -182,7 +182,7 @@ def geoserver_post_save(instance, sender, **kwargs):
 
     if instance.storeType == "remoteStore":
         # Save layer attributes
-        set_attributes(instance)
+        set_attributes_from_geoserver(instance)
         return
 
     if not getattr(instance, 'gs_resource', None):
@@ -471,7 +471,7 @@ def command_url(command):
         Link.objects.filter(pk=link.pk).update(url=tile_url)
 
     # Save layer attributes
-    set_attributes(instance)
+    set_attributes_from_geoserver(instance)
 
     # Save layer styles
     set_styles(instance, gs_catalog)

geonode/services/views.py

@@ -57,7 +57,7 @@
 from geonode.services.forms import CreateServiceForm, ServiceForm
 from geonode.utils import mercator_to_llbbox
 from geonode.layers.utils import create_thumbnail
-from geonode.geoserver.helpers import set_attributes
+from geonode.geoserver.helpers import set_attributes_from_geoserver
 from geonode.base.models import Link
 
 logger = logging.getLogger("geonode.core.layers.views")
@@ -655,7 +655,7 @@ def _register_indexed_layers(service, wms=None, verbosity=False):
                 saved_layer.save()
                 saved_layer.set_default_permissions()
                 saved_layer.keywords.add(*keywords)
-                set_attributes(saved_layer)
+                set_attributes_from_geoserver(saved_layer)
 
                 service_layer, created = ServiceLayer.objects.get_or_create(
                     typename=wms_layer.name,

geonode/tests/integration.py

@@ -48,7 +48,7 @@
 )
 from geonode.tests.utils import check_layer, get_web_page
 
-from geonode.geoserver.helpers import cascading_delete, set_attributes
+from geonode.geoserver.helpers import cascading_delete, set_attributes_from_geoserver
 # FIXME(Ariel): Uncomment these when #1767 is fixed
 # from geonode.geoserver.helpers import get_time_info
 # from geonode.geoserver.helpers import get_wms
@@ -1001,7 +1001,7 @@ def setUp(self):
     def tearDown(self):
         pass
 
-    def test_set_attributes(self):
+    def test_set_attributes_from_geoserver(self):
         """Test attributes syncronization
         """
 
@@ -1018,7 +1018,7 @@ def test_set_attributes(self):
             attribute.save()
 
         # sync the attributes with GeoServer
-        set_attributes(layer)
+        set_attributes_from_geoserver(layer)
 
         # tests if everything is synced properly
         for attribute in layer.attribute_set.all():

geonode/tests/utils.py

@@ -18,10 +18,18 @@
 #
 #########################################################################
 
+import contextlib
+import copy
 import urllib
 import urllib2
-import contextlib
+
+from django.core.management import call_command
+from django.db.models import signals
+from django.test import TestCase
+
+from geonode.geoserver.signals import geoserver_post_save
 from geonode.maps.models import Layer
+from geonode.utils import set_attributes
 
 
 def get_web_page(url, username=None, password=None, login_url=None):
@@ -87,3 +95,49 @@ def check_layer(uploaded):
     assert isinstance(uploaded, Layer), msg
     msg = ('The layer does not have a valid name: %s' % uploaded.name)
     assert len(uploaded.name) > 0, msg
+
+
+class TestSetAttributes(TestCase):
+
+    def setUp(self):
+        # Load users to log in as
+        call_command('loaddata', 'people_data', verbosity=0)
+
+    def test_set_attributes_creates_attributes(self):
+        """ Test utility function set_attributes() which creates Attribute instances attached
+            to a Layer instance.
+        """
+        # Creating a layer requires being logged in
+        self.client.login(username='norman', password='norman')
+
+        # Disconnect the geoserver-specific post_save signal attached to Layer creation.
+        # The geoserver signal handler assumes things about the store where the Layer is placed.
+        # this is a workaround.
+        disconnected_post_save = signals.post_save.disconnect(geoserver_post_save, sender=Layer)
+
+        # Create dummy layer to attach attributes to
+        l = Layer.objects.create(name='dummy_layer')
+
+        # Reconnect the signal if it was disconnected
+        if disconnected_post_save:
+            signals.post_save.connect(geoserver_post_save, sender=Layer)
+
+        attribute_map = [
+            ['id', 'Integer'],
+            ['date', 'IntegerList'],
+            ['enddate', 'Real'],
+            ['date_as_date', 'xsd:dateTime'],
+        ]
+
+        # attribute_map gets modified as a side-effect of the call to set_attributes()
+        expected_results = copy.deepcopy(attribute_map)
+
+        # set attributes for resource
+        set_attributes(l, attribute_map)
+
+        # 2 items in attribute_map should translate into 2 Attribute instances
+        self.assertEquals(l.attributes.count(), len(expected_results))
+
+        # The name and type should be set as provided by attribute map
+        for a in l.attributes:
+            self.assertIn([a.attribute, a.attribute_type], expected_results)