erge branch 'master' into wm-develop

.gitignore

@@ -11,6 +11,7 @@ dist
 geonode/development.db
 /geoserver/
 downloaded
+package-lock.json
 
 docs/i18n/pot/
 

dev_config.yml

@@ -1,7 +1,7 @@
 ---
 GEOSERVER_URL: "http://build.geonode.org/geoserver/latest/geoserver-2.10.x.war"
 DATA_DIR_URL: "http://build.geonode.org/geoserver/latest/data-2.10.x.zip"
-JETTY_RUNNER_URL: "http://repo2.maven.org/maven2/org/mortbay/jetty/jetty-runner/8.1.8.v20121106/jetty-runner-8.1.8.v20121106.jar"
+JETTY_RUNNER_URL: "http://repo2.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.7.v20170914/jetty-runner-9.4.7.v20170914.jar"
 WINDOWS:
   py2exe: "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe"
   nose: "https://s3.amazonaws.com/geonodedeps/nose-1.3.3.win32-py2.7.exe"

docs/tutorials/users/managing_layers/layer_filter.txt

@@ -0,0 +1,19 @@
+.. _layers.detail:
+
+Download a layer filtered by attributes
+=====================
+
+In GeoNode it is possible to download parts of a layer using a Query Builder. The Query Builder provides a step by step graphical environment, where the user is able to select attributes existing in the layer (e.g. population of an area) and set numerical or text filters.
+
+The filters can be chained together either by an AND or an OR operator. Moreover when building the Query statements, the user can retrieve all the available values for each attribute. This is particularly useful, when the user wants to filter data based on numbers, locations etc. which he/she is not familiarized.
+
+.. figure:: img/layer_filtering.png
+
+Access the Layer Filtering:
+
+* select 'Layers' from the main menu.
+* choose the desired layer by clicking on its title.
+* click the button 'Download Layer' (on the upper right part of the screen)
+* choose "Data" tab and select the button: 'Do you want to filter it?'
+* use the filtering functionality to build the CQL Query.
+* pick up one of the format you want to download the layer

geonode/contrib/datastore_shards/models.py

@@ -52,9 +52,11 @@ def update_shard_layers_count(instance, sender, **kwargs):
     Update layers_count for Database model.
     """
     store_name = instance.store
-    shardatabase = Database.objects.get(name=store_name)
-    shardatabase.layers_count = Layer.objects.filter(store=store_name).count()
-    shardatabase.save()
+    # if layer is part of a shards we need to increment layers_count
+    if Database.objects.filter(name=store_name).exists():
+        shardatabase = Database.objects.get(name=store_name)
+        shardatabase.layers_count = Layer.objects.filter(store=store_name).count()
+        shardatabase.save()
 
 
 signals.post_delete.connect(update_shard_layers_count, sender=Layer)

geonode/geoserver/views.py

@@ -141,8 +141,11 @@ def respond(*args, **kw):
     el = dom.findall(
         "{http://www.opengis.net/sld}NamedLayer/{http://www.opengis.net/sld}Name")
     if len(el) == 0 and not data.get('name'):
-        return respond(
-            errors="Please provide a name, unable to extract one from the SLD.")
+        el = dom.findall(
+            "{http://www.opengis.net/sld}UserLayer/{http://www.opengis.net/sld}Name")
+        if len(el) == 0 and not data.get('name'):
+            return respond(
+                errors="Please provide a name, unable to extract one from the SLD.")
     name = data.get('name') or el[0].text
     if data['update']:
         match = None

geonode/layers/urls.py

@@ -31,6 +31,7 @@
     url(r'^upload$', 'layer_upload', name='layer_upload'),
     url(r'^upload_metadata$', 'layer_metadata_upload', name='layer_metadata_upload'),
     url(r'^upload_style$', 'layer_sld_upload', name='layer_sld_upload'),
+    url(r'^load_layer_data$', 'load_layer_data', name='load_layer_data'),
     url(r'^(?P<layername>[^/]*)$', 'layer_detail', name="layer_detail"),
     url(r'^(?P<layername>[^/]*)/metadata$', 'layer_metadata', name="layer_metadata"),
     url(r'^(?P<layername>[^/]*)/metadata_advanced$', 'layer_metadata_advanced', name="layer_metadata_advanced"),

geonode/layers/views.py

@@ -29,6 +29,9 @@
 from lxml import etree
 from requests import Request
 from itertools import chain
+from six import string_types
+from owslib.wfs import WebFeatureService
+from owslib.feature.schema import get_schema
 
 from guardian.shortcuts import get_perms
 from django.contrib import messages
@@ -428,14 +431,101 @@ def layer_detail(request, layername, template='layers/layer_detail.html'):
 
     if settings.SOCIAL_ORIGINS:
         context_dict["social_links"] = build_social_links(request, layer)
+    layers_names = layer.alternate
+    try:
+        if 'geonode' in layers_names:
+            workspace, name = layers_names.split(':', 1)
+        else:
+            name = layers_names
+    except:
+        print "Can not identify workspace type and layername"
+
+    context_dict["layer_name"] = json.dumps(layers_names)
+
+    try:
+        # get type of layer (raster or vector)
+        if layer.storeType == 'coverageStore':
+            context_dict["layer_type"] = "raster"
+        elif layer.storeType == 'dataStore':
+            context_dict["layer_type"] = "vector"
+
+            location = "{location}{service}".format(** {
+                'location': settings.OGC_SERVER['default']['LOCATION'],
+                'service': 'wms',
+            })
+            # get schema for specific layer
+            username = settings.OGC_SERVER['default']['USER']
+            password = settings.OGC_SERVER['default']['PASSWORD']
+            schema = get_schema(location, name, username=username, password=password)
+
+            # get the name of the column which holds the geometry
+            if 'the_geom' in schema['properties']:
+                schema['properties'].pop('the_geom', None)
+            elif 'geom' in schema['properties']:
+                schema['properties'].pop("geom", None)
+
+            # filter the schema dict based on the values of layers_attributes
+            layer_attributes_schema = []
+            for key in schema['properties'].keys():
+                    layer_attributes_schema.append(key)
+
+            filtered_attributes = layer_attributes_schema
+            context_dict["schema"] = schema
+            context_dict["filtered_attributes"] = filtered_attributes
+
+    except:
+        print "Possible error with OWSLib. Turning all available properties to string"
 
     # maps owned by user needed to fill the "add to existing map section" in template
     if request.user.is_authenticated():
         context_dict["maps"] = Map.objects.filter(owner=request.user)
-
     return render_to_response(template, RequestContext(request, context_dict))
 
 
+# Loads the data using the OWS lib when the "Do you want to filter it" button is clicked.
+def load_layer_data(request, template='layers/layer_detail.html'):
+    context_dict = {}
+    data_dict = json.loads(request.POST.get('json_data'))
+    layername = data_dict['layer_name']
+    filtered_attributes = data_dict['filtered_attributes']
+    workspace, name = layername.split(':')
+    location = "{location}{service}".format(** {
+        'location': settings.OGC_SERVER['default']['LOCATION'],
+        'service': 'wms',
+    })
+
+    try:
+        username = settings.OGC_SERVER['default']['USER']
+        password = settings.OGC_SERVER['default']['PASSWORD']
+        wfs = WebFeatureService(location, version='1.1.0', username=username, password=password)
+        response = wfs.getfeature(typename=name, propertyname=filtered_attributes, outputFormat='application/json')
+        x = response.read()
+        x = json.loads(x)
+        features_response = json.dumps(x)
+        decoded = json.loads(features_response)
+        decoded_features = decoded['features']
+        properties = {}
+        for key in decoded_features[0]['properties']:
+            properties[key] = []
+
+        # loop the dictionary based on the values on the list and add the properties
+        # in the dictionary (if doesn't exist) together with the value
+        for i in range(len(decoded_features)):
+
+            for key, value in decoded_features[i]['properties'].iteritems():
+                if value != '' and isinstance(value, (string_types, int, float)):
+                    properties[key].append(value)
+
+        for key in properties:
+            properties[key] = list(set(properties[key]))
+            properties[key].sort()
+
+        context_dict["feature_properties"] = properties
+    except:
+        print "Possible error with OWSLib."
+    return HttpResponse(json.dumps(context_dict), content_type="application/json")
+
+
 def layer_feature_catalogue(
         request,
         layername,

geonode/maps/views.py

@@ -382,6 +382,7 @@ def map_embed(
 
 # MAPS VIEWER #
 
+
 @require_http_methods(["GET", ])
 def add_layer(request):
     """
@@ -397,24 +398,15 @@ def add_layer(request):
         'base.view_resourcebase',
         _PERMISSION_MSG_VIEW)
 
-    config = add_layers_to_map_config(request, map_obj, (layer_name, ))
-
-    return render_to_response('maps/map_edit.html', RequestContext(request, {
-        'mapId': map_id,
-        'config': json.dumps(config),
-        'map': map_obj,
-        'preview': getattr(
-            settings,
-            'LAYER_PREVIEW_LIBRARY',
-            '')
-    }))
+    return map_view(request, str(map_obj.id), layer_name=layer_name)
 
 
-def map_view(request, mapid, snapshot=None, template='maps/map_view.html'):
+def map_view(request, mapid, snapshot=None, layer_name=None, template='maps/map_view.html'):
     """
     The view that returns the map composer opened to
     the map with the given map ID.
     """
+
     map_obj = _resolve_map(
         request,
         mapid,
@@ -431,6 +423,9 @@ def map_view(request, mapid, snapshot=None, template='maps/map_view.html'):
     else:
         config = snapshot_config(snapshot, map_obj, request.user, access_token)
 
+    if layer_name:
+        config = add_layers_to_map_config(request, map_obj, (layer_name, ), False)
+
     return render_to_response(template, RequestContext(request, {
         'config': json.dumps(config),
         'map': map_obj,
@@ -685,7 +680,7 @@ def new_map_config(request):
     return json.dumps(config)
 
 
-def add_layers_to_map_config(request, map_obj, layer_names):
+def add_layers_to_map_config(request, map_obj, layer_names, add_base_layers=True):
     DEFAULT_MAP_CONFIG, DEFAULT_BASE_LAYERS = default_map_config(request)
     if 'access_token' in request.session:
         access_token = request.session['access_token']
@@ -809,8 +804,14 @@ def add_layers_to_map_config(request, map_obj, layer_names):
         map_obj.zoom = math.ceil(min(width_zoom, height_zoom))
 
     map_obj.handle_moderated_uploads()
+
+    if add_base_layers:
+        layers_to_add = DEFAULT_BASE_LAYERS + layers
+    else:
+        layers_to_add = layers
     config = map_obj.viewer_json(
-        request.user, access_token, *(DEFAULT_BASE_LAYERS + layers))
+        request.user, access_token, *layers_to_add)
+
     config['fromLayer'] = True
 
     return config

geonode/messaging/consumer.py

@@ -75,12 +75,12 @@ def on_consume_end(self, connection, channel):
         logger.info("finished.")
 
     def on_message(self, body, message):
-        logger.info("broadcast: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("broadcast: RECEIVED MSG - body: %r" % (body,))
         message.ack()
         self._check_message_limit()
 
     def on_email_messages(self, body, message):
-        logger.info("on_email_messages: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("on_email_messages: RECEIVED MSG - body: %r" % (body,))
         layer_uuid = body.get("layer_uuid")
         user_id = body.get("user_id")
         send_email_consumer(layer_uuid, user_id)
@@ -90,7 +90,7 @@ def on_email_messages(self, body, message):
         self._check_message_limit()
 
     def on_geoserver_messages(self, body, message):
-        logger.info("on_geoserver_messages: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("on_geoserver_messages: RECEIVED MSG - body: %r" % (body,))
         layer_id = body.get("id")
         geoserver_post_save_local(layer_id)
         # Not sure if we need to send ack on this fanout version.
@@ -99,7 +99,7 @@ def on_geoserver_messages(self, body, message):
         self._check_message_limit()
 
     def on_notifications_messages(self, body, message):
-        logger.info("on_notifications_message: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("on_notifications_message: RECEIVED MSG - body: %r" % (body,))
         body.get("id")
         body.get("app_label")
         body.get("model")
@@ -110,20 +110,20 @@ def on_notifications_messages(self, body, message):
         self._check_message_limit()
 
     def on_geoserver_all(self, body, message):
-        logger.info("on_geoserver_all: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("on_geoserver_all: RECEIVED MSG - body: %r" % (body,))
         message.ack()
         logger.info("on_geoserver_all: finished")
         # TODO:Adding consurmer's producers.
         self._check_message_limit()
 
     def on_geoserver_catalog(self, body, message):
-        logger.info("on_geoserver_catalog: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("on_geoserver_catalog: RECEIVED MSG - body: %r" % (body,))
         message.ack()
         logger.info("on_geoserver_catalog: finished")
         self._check_message_limit()
 
     def on_geoserver_data(self, body, message):
-        logger.info("on_geoserver_data: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("on_geoserver_data: RECEIVED MSG - body: %r" % (body,))
         message.ack()
         logger.info("on_geoserver_data: finished")
         self._check_message_limit()
@@ -138,7 +138,7 @@ def on_consume_ready(self, connection, channel, consumers, **kwargs):
                                                **kwargs)
 
     def on_layer_viewer(self, body, message):
-        logger.info("on_layer_viewer: RECEIVED MSG - body: %r" % (body,))
+        logger.debug("on_layer_viewer: RECEIVED MSG - body: %r" % (body,))
         viewer = body.get("viewer")
         owner_layer = body.get("owner_layer")
         layer_id = body.get("layer_id")